Support ignoring certain rows

Signed-off-by: Roberto Rosario <roberto.rosario.gonzalez@gmail.com>
This commit is contained in:
Roberto Rosario
2019-06-20 10:12:53 -04:00
parent 5808d3653d
commit 925b55d76d

View File

@@ -34,6 +34,11 @@ class Command(management.BaseCommand):
help='Don\'t stop the import process on common errors like ' help='Don\'t stop the import process on common errors like '
'incorrect file paths.', 'incorrect file paths.',
) )
parser.add_argument(
'--ignore_rows',
action='store', dest='ignore_rows', default='',
help='Ignore a set of rows. Row numbers must be separated by commas.'
)
parser.add_argument( parser.add_argument(
'--metadata_pairs_column', '--metadata_pairs_column',
action='store', dest='metadata_pairs_column', action='store', dest='metadata_pairs_column',
@@ -49,6 +54,10 @@ class Command(management.BaseCommand):
document_types = {} document_types = {}
uploaded_count = 0 uploaded_count = 0
row_count = 0 row_count = 0
rows_to_ignore = []
for entry in options['ignore_rows'].split(','):
if entry:
rows_to_ignore.append(int(entry))
DocumentType = apps.get_model( DocumentType = apps.get_model(
app_label='documents', model_name='DocumentType' app_label='documents', model_name='DocumentType'
@@ -64,7 +73,11 @@ class Command(management.BaseCommand):
with open(options['filelist']) as csv_datafile: with open(options['filelist']) as csv_datafile:
csv_reader = csv.reader(csv_datafile) csv_reader = csv.reader(csv_datafile)
for row in csv_reader: for row in csv_reader:
# Increase row count here even though start index is 0
# purpose is to avoid losing row number increments on
# exceptions
row_count = row_count + 1 row_count = row_count + 1
if row_count - 1 not in rows_to_ignore:
try: try:
with open(row[options['document_path_column']]) as file_object: with open(row[options['document_path_column']]) as file_object:
document_type_label = row[options['document_type_column']] document_type_label = row[options['document_type_column']]