diff --git a/events/management/commands/load_cities.py b/events/management/commands/load_cities.py index 79880fb..c571969 100644 --- a/events/management/commands/load_cities.py +++ b/events/management/commands/load_cities.py @@ -1,31 +1,32 @@ -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from events.models.locale import Country, SPR, City # Fields from geoname table, from http://download.geonames.org/export/dump/readme.txt -GEONAMEID=0 -NAME=1 -ASCIINAME=2 -ALTERNATENAMES=3 -LATITUDE=4 -LONGITUDE=5 -FEATURE_CLASS=6 -FEATURE_CODE=7 -COUNTRY_CODE=8 -COUNTRY_CODE_2=9 -ADMIN1=10 -ADMIN2=11 -ADMIN3=12 -ADMIN4=13 -POPULATION=14 -ELEVATION=15 -DIGITAL_ELEVATION=16 -TIMEZONE=17 -MODIFICATION_DATE=18 +GEONAMEID = 0 +NAME = 1 +ASCIINAME = 2 +ALTERNATENAMES = 3 +LATITUDE = 4 +LONGITUDE = 5 +FEATURE_CLASS = 6 +FEATURE_CODE = 7 +COUNTRY_CODE = 8 +COUNTRY_CODE_2 = 9 +ADMIN1 = 10 +ADMIN2 = 11 +ADMIN3 = 12 +ADMIN4 = 13 +POPULATION = 14 +ELEVATION = 15 +DIGITAL_ELEVATION = 16 +TIMEZONE = 17 +MODIFICATION_DATE = 18 COUNTRY_CACHE = dict() SPR_CACHE = dict() + class Command(BaseCommand): help = 'Loads city data from GeoNames database file' @@ -38,22 +39,24 @@ class Command(BaseCommand): for country in Country.objects.all(): COUNTRY_CACHE[country.code] = country for spr in SPR.objects.all(): - SPR_CACHE["%s.%s"%(spr.country.code, spr.code)] = spr - cities_file = open(options['file'], 'r') - for city_line in cities_file.readlines(): - city = city_line.split("\t") - if len(city) == 19: - if city[FEATURE_CODE].startswith("PPL"): - country = COUNTRY_CACHE.get(city[COUNTRY_CODE]) - spr = SPR_CACHE.get("%s.%s"%(city[COUNTRY_CODE], city[ADMIN1])) - if country is not None and spr is not None: - try: - City.objects.update_or_create(name=city[NAME], spr=spr, defaults={'tz':city[TIMEZONE], 'longitude':city[LONGITUDE], 'latitude':city[LATITUDE]}) - except: - print("Warning: Failed to load city %s for %s" % (city[NAME], spr)) - else: - print("Short line (%s): %s" % (len(city), city_line)) - cities_file.close() + SPR_CACHE["%s.%s" % (spr.country.code, spr.code)] = spr + with open(options['file'], 'r') as cities_file: + for city_line in cities_file.readlines(): + city = city_line.split("\t") + if len(city) == 19: + if city[FEATURE_CODE].startswith("PPL"): + country = COUNTRY_CACHE.get(city[COUNTRY_CODE]) + spr = SPR_CACHE.get("%s.%s" % (city[COUNTRY_CODE], city[ADMIN1])) + if country is not None and spr is not None: + try: + City.objects.update_or_create( + name=city[NAME], + spr=spr, + defaults={'tz': city[TIMEZONE], 'longitude': city[LONGITUDE], 'latitude': city[LATITUDE]}) + except Exception as e: + print("Warning: Failed to load city %s for %s (%s)" % (city[NAME], spr, e)) + else: + print("Short line (%s): %s" % (len(city), city_line)) else: print("No File in options!") diff --git a/events/management/commands/load_countries.py b/events/management/commands/load_countries.py index df73af4..f5d652b 100644 --- a/events/management/commands/load_countries.py +++ b/events/management/commands/load_countries.py @@ -1,27 +1,28 @@ -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from events.models.locale import Country # Fields from geoname table, from http://download.geonames.org/export/dump/readme.txt -ISO=0 -ISO3=1 -ISO_NUMERIC=2 -FIPS=3 -COUNTRY=4 -CAPITAL=5 -AREA=6 -POPULATION=7 -CONTINENT=8 -TLD=9 -CURRENCYCODE=10 -CURRENCYNAME=11 -PHONE=12 -POSTAL_CODE_FORMAT=13 -POSTAL_CODE_REGEX=14 -LANGUAGES=15 -GEONAMEID=16 -NEIGHBOURS=17 -EQUIVALENTFIPSCODE=18 +ISO = 0 +ISO3 = 1 +ISO_NUMERIC = 2 +FIPS = 3 +COUNTRY = 4 +CAPITAL = 5 +AREA = 6 +POPULATION = 7 +CONTINENT = 8 +TLD = 9 +CURRENCYCODE = 10 +CURRENCYNAME = 11 +PHONE = 12 +POSTAL_CODE_FORMAT = 13 +POSTAL_CODE_REGEX = 14 +LANGUAGES = 15 +GEONAMEID = 16 +NEIGHBOURS = 17 +EQUIVALENTFIPSCODE = 18 + class Command(BaseCommand): help = 'Loads country data from GeoNames database file' @@ -31,17 +32,15 @@ class Command(BaseCommand): def handle(self, *args, **options): if 'file' in options: - countries_file = open(options['file'], 'r') - for country_line in countries_file.readlines(): - if country_line.startswith("#"): - continue - country = country_line.split("\t") - if len(country) == 19: - #print("%s - %s" % (country[ISO], country[COUNTRY])) - Country.objects.get_or_create(name=country[COUNTRY], code=country[ISO]) - else: - print("Short line (%s): %s" % (len(country), country_line)) - countries_file.close() - + with open(options['file'], 'r') as countries_file: + for country_line in countries_file.readlines(): + if country_line.startswith("#"): + continue + country = country_line.split("\t") + if len(country) == 19: + # print("%s - %s" % (country[ISO], country[COUNTRY])) + Country.objects.get_or_create(name=country[COUNTRY], code=country[ISO]) + else: + print("Short line (%s): %s" % (len(country), country_line)) else: print("No File in options!") diff --git a/events/management/commands/load_spr.py b/events/management/commands/load_spr.py index 61e4de2..8e2a6f5 100644 --- a/events/management/commands/load_spr.py +++ b/events/management/commands/load_spr.py @@ -3,10 +3,10 @@ from django.core.management.base import BaseCommand, CommandError from events.models.locale import Country, SPR, City # Fields from geoname table, from http://download.geonames.org/export/dump/readme.txt -COMBINED_CODE=0 -NAME=1 -ASCIINAME=2 -GEONAMEID=3 +COMBINED_CODE = 0 +NAME = 1 +ASCIINAME = 2 +GEONAMEID = 3 COUNTRY_CACHE = dict() @@ -22,20 +22,18 @@ class Command(BaseCommand): # Preload country cache for country in Country.objects.all(): COUNTRY_CACHE[country.code] = country - spr_file = open(options['file'], 'r') - for spr_line in spr_file.readlines(): - if spr_line.startswith("#"): - continue - spr = spr_line.split("\t") - if len(spr) ==4: - COUNTRY_CODE, SPR_CODE = spr[COMBINED_CODE].split(".") - country = COUNTRY_CACHE.get(COUNTRY_CODE) - if country is not None: - #print("%s - %s, %s" % (SPR_CODE, spr[NAME], country.name)) - SPR.objects.get_or_create(name=spr[NAME], code=SPR_CODE, country=country) - else: - print("Short line (%s): %s" % (len(spr), spr_line)) - spr_file.close() - + with open(options['file'], 'r') as spr_file: + for spr_line in spr_file.readlines(): + if spr_line.startswith("#"): + continue + spr = spr_line.split("\t") + if len(spr) ==4: + COUNTRY_CODE, SPR_CODE = spr[COMBINED_CODE].split(".") + country = COUNTRY_CACHE.get(COUNTRY_CODE) + if country is not None: + #print("%s - %s, %s" % (SPR_CODE, spr[NAME], country.name)) + SPR.objects.get_or_create(name=spr[NAME], code=SPR_CODE, country=country) + else: + print("Short line (%s): %s" % (len(spr), spr_line)) else: print("No File in options!")