Add primary fuel type indicator & replace fuel1-4

Replace database fields `fuel1`, `fuel2`, `fuel3`, `fuel4` with
fields `primary_fuel`, `other_fuel1`, `other_fuel2`, `other_fuel3`.

Unlike previous database versions, which did not place any priority
on the ordering of the fuels, `primary_fuel` is the fuel that has
precendence over `other_fuel*`, though each `other_fuel` is not
ranked in priority over any `other_fuel`.

When unit-level information is available, the primary fuel is determined
to be the fuel that has the greatest share of capacity assigned to it.

Though there was no meaningful priority in `fuel*`, there were some changes in the
ordering of fuels with this update.

880 plants (265 GW) have a `primary_fuel` that is not equal to `fuel1`.
849 of these are in the USA (241.66 GW).
5 plants (400 MW) have a `primary_fuel` that was not one of `fuel*` (all
in the USA, all are now Biomass plants).
438 plants (215 GW) have additional fuel information that was not in the
database previously.
These plants are mostly from the Fusion Tables, which is now reading in
both the Fuel and Secondary Fuel fields.

Net change: -8 plants @ +126 MW

ARG: -10 plants @ 0 MW
Some units were being read as individual plants. In this update, these
units are all read into a single plant, which has the lowest ID of all
the units from the former database.
	ALTO VALLE
		ARG0000235 (combined into this ID)
		ARG0000236 REMOVED
		ARG0000237 REMOVED
		ARG0000239 REMOVED
		ARG0000238 REMOVED
	LUJAN DE CUYO
		ARG0000198 (combined into this ID)
		ARG0000199 REMOVED
		ARG0000200 REMOVED
		ARG0000201 REMOVED
		ARG0000202 REMOVED
		ARG0000203 REMOVED
	COMODORO RIVADAVIA
		ARG0000073 (combined into this ID)
		ARG0000074 REMOVED

CHL: +1 plant @ +63 MW
ESP: +1 plant @ +63 MW

Database Version: 1.1.9

build_databases/build_database_ARG.py

@@ -30,8 +30,7 @@
 SOURCE_URL = u"http://energia3.mecon.gov.ar/contenidos/archivos/Reorganizacion/informacion_del_mercado/publicaciones/mercado_electrico/estadisticosectorelectrico/2015/A1.POT_GEN_COMB_POR_CENTRAL_2015.xlsx"
 SAVE_CODE = u"ARG"
 RAW_FILE_NAME = pw.make_file_path(fileType="raw", subFolder=SAVE_CODE, filename="A1.POT_GEN_COMB_POR_CENTRAL_2015.xlsx")
-LOCATION_FILE_NAME = pw.make_file_path(fileType="resource", subFolder=SAVE_CODE, filename="locations_ARG.csv")
-COMMISSIONING_YEAR_FILE_NAME = pw.make_file_path(fileType="resource", subFolder=SAVE_CODE, filename="commissioning_years_ARG.csv")
+PLANT_AUX_FILE = pw.make_file_path(fileType="resource", subFolder=SAVE_CODE, filename="ARG_plants.csv")
 CSV_FILE_NAME = pw.make_file_path(fileType = "src_csv", filename = "database_ARG.csv")
 SAVE_DIRECTORY = pw.make_file_path(fileType = "src_bin")
 YEAR_OF_DATA = 2015
@@ -58,147 +57,125 @@
 # extract powerplant information from file(s)
 print(u"Reading in plants...")
 
-# read locations
-locations_dictionary = {}
-with open(LOCATION_FILE_NAME, 'r') as f:
-    datareader = csv.reader(f)
-    headers = datareader.next()
-    for row in datareader:
-        locations_dictionary[pw.format_string(row[0])] = [float(row[1]), float(row[2])]
-
-# read commissioning years
-commissioning_years_dictionary = {}
-with open(COMMISSIONING_YEAR_FILE_NAME, 'r') as f:
-    datareader = csv.reader(f)
-    headers = datareader.next()
-    for row in datareader:
-        commissioning_years_dictionary[pw.format_string(row[0])] = row[1]
+# read auxilliary plant information
+with open(PLANT_AUX_FILE, 'r') as f:
+    reader = csv.DictReader(f)
+    aux_plant_info = {pw.format_string(row['name']): row for row in reader}
 
 # read data from csv and parse
 count = 1
 
 wb = xlrd.open_workbook(RAW_FILE_NAME)
 ws = wb.sheet_by_name(TAB)
 
-# treat first data row specially for plant name
-rv0 = ws.row_values(START_ROW)
-current_plant_name = pw.format_string(rv0[COLS['name']])
-current_owner = pw.format_string(rv0[COLS['owner']])
-current_fuel_types = pw.standardize_fuel(rv0[COLS['fuel']], fuel_thesaurus)
-current_capacity_sum = float(rv0[COLS['capacity']]) * CAPACITY_CONVERSION_TO_MW
-current_generation_sum = float(rv0[COLS['generation']]) * GENERATION_CONVERSION_TO_GWH
+previous_owner = u'None'
+previous_name = u'None'
+plant_names = {}
 
-test_print = False
-for row_id in range(START_ROW+1, ws.nrows):
-    rv = ws.row_values(row_id) 
+for row_id in range(START_ROW, ws.nrows):
 
-    row_fuel = pw.format_string(rv[COLS['fuel']], None)
-    row_name = pw.format_string(rv[COLS['name']], None)
-    row_grid = pw.format_string(rv[COLS['grid']], None)
+    rv = ws.row_values(row_id) 
 
-    if row_grid == u"AISLADO":
+    # check for islanded generator
+    grid_string = pw.format_string(rv[COLS['grid']], None)
+    if grid_string == u"AISLADO":
         continue                    # don't add islanded generators (not grid-connected)
 
-    if row_fuel:
-
-        if row_name:
-
-            if current_plant_name:
-
-                # assign ID number, make PowerPlant object, add to dictionary
-                idnr = pw.make_id(SAVE_CODE, count)
-                annual_generation = pw.PlantGenerationObject(gwh=current_generation_sum, start_date=gen_start, end_date=gen_stop, source=SOURCE_NAME)
-                new_plant = pw.PowerPlant(plant_idnr=idnr, plant_name=current_plant_name, plant_owner=current_owner,
-                    plant_fuel=current_fuel_types, plant_country=COUNTRY_NAME, plant_capacity=current_capacity_sum,
-                    plant_cap_year=YEAR_OF_DATA, plant_source=SOURCE_NAME, plant_source_url=SOURCE_URL,
-                    plant_generation=annual_generation)
-                plants_dictionary[idnr] = new_plant
-                count += 1
-
-            # reset all current values to this row
-            current_plant_name = row_name
-            current_fuel_types = pw.standardize_fuel(row_fuel, fuel_thesaurus)
-            current_owner = pw.format_string(rv[COLS['owner']], None)
-            try:
-                current_capacity_sum = float(rv[COLS['capacity']]) * CAPACITY_CONVERSION_TO_MW
-            except:
-                print(u"-Error: Can't read capacity for plant {0}.".format(current_plant_name))
-                current_capacity_sum = 0.0
-            try:
-                current_generation_sum = float(rv[COLS['generation']]) * GENERATION_CONVERSION_TO_GWH
-            except:
-                print(u"-Error: Can't read generation for plant {0}; value is {1}.".format(current_plant_name, rv[COLS['generation']]))
-                current_generation_sum = 0.0
-
-        else:
-            # additional unit of current plant
-            current_capacity_sum += float(rv[COLS['capacity']]) * CAPACITY_CONVERSION_TO_MW
-            current_generation_sum += float(rv[COLS['generation']]) * GENERATION_CONVERSION_TO_GWH
-            current_fuel_types.update(pw.standardize_fuel(row_fuel, fuel_thesaurus))
+    # get fuel
+    fuel_string = pw.format_string(rv[COLS['fuel']], None)
+    if not fuel_string:
+        continue                    # row without fuel type is empty
+
+    # get name
+    name_string = pw.format_string(rv[COLS['name']], None)
+    if name_string:
+        previous_name = name_string
+    else:
+        name_string = previous_name
+
+    if name_string not in aux_plant_info:
+        print("Can't find plant <{0}> in auxiliary plant information file, skipping..".format(name_string))
+        continue
 
+    # get owner
+    owner_string = pw.format_string(rv[COLS['owner']], None)
+    if owner_string:
+        previous_owner = owner_string
     else:
+        owner_string = previous_owner
+
+    # get capacity
+    try:
+        capacity_value = float(rv[COLS['capacity']]) * CAPACITY_CONVERSION_TO_MW
+    except:
+        print("Cant read capacity for plant {0}.".format(name_string))
+        capacity_value = 0
+
+    # check if we've seen this plant before
+    if name_string not in plant_names:
+        # first time we've seen this plant
+        fuel_type = pw.standardize_fuel(fuel_string, fuel_thesaurus, as_set=False)
+        idnr = pw.format_string(aux_plant_info[name_string]['gppd_idnr'])
+        new_plant = pw.PowerPlant(plant_idnr=idnr, plant_name=name_string, plant_owner=owner_string,
+            plant_country=COUNTRY_NAME, plant_capacity=capacity_value,
+            plant_primary_fuel=fuel_type,
+            plant_cap_year=YEAR_OF_DATA, plant_source=SOURCE_NAME, 
+            plant_source_url=SOURCE_URL)
+        plants_dictionary[idnr] = new_plant
+        plant_names[name_string] = (idnr, {fuel_type: capacity_value})
+        # increment count
+        count += 1
 
-        if current_plant_name:
+    else:
 
-            # assign ID number, make PowerPlant object, add to dictionary
-            idnr = pw.make_id(SAVE_CODE, count)
-            annual_generation = pw.PlantGenerationObject(gwh=current_generation_sum, start_date=gen_start, end_date=gen_stop, source=SOURCE_NAME)
-            new_plant = pw.PowerPlant(plant_idnr=idnr, plant_name=current_plant_name, plant_owner=current_owner,
-                plant_fuel=current_fuel_types, plant_country=COUNTRY_NAME, plant_capacity=current_capacity_sum,
-                plant_cap_year=YEAR_OF_DATA, plant_source=SOURCE_NAME, plant_source_url=SOURCE_URL,
-                plant_generation=annual_generation)
-            plants_dictionary[idnr] = new_plant
-            count += 1
+        # this row is an additional fuel type for a plant we've already seen
+        fuel_type = pw.standardize_fuel(fuel_string, fuel_thesaurus, as_set=False)
+        idnr, fuel_capacity_dict = plant_names[name_string]
+        fuel_capacity = fuel_capacity_dict.get(fuel_type, 0)
+        fuel_capacity += capacity_value
+        fuel_capacity_dict[fuel_type] = fuel_capacity
+        plants_dictionary[idnr].capacity += capacity_value
 
-            # reset all current values to null
-            current_plant_name = pw.NO_DATA_UNICODE
-            current_fuel_types = pw.NO_DATA_SET.copy()
-            current_owner = pw.NO_DATA_UNICODE
-            current_capacity_sum = 0.0
-            current_generation_sum = 0.0
 
-        else:
-            continue
+# assign primary and other fuel types to each plant
+for name, (idnr, fuel_capacity_dict) in plant_names.iteritems():
+    primary_fuel = max(fuel_capacity_dict, key=lambda x: fuel_capacity_dict[x])
+    other_fuels = set(fuel_capacity_dict.keys())
+    other_fuels.remove(primary_fuel)
+    plants_dictionary[idnr].primary_fuel = primary_fuel
+    plants_dictionary[idnr].other_fuel = other_fuels
 
 # now assign locations and commissioning years
 location_not_found = 0
 year_not_found = 0
 
-for idnr,plant in plants_dictionary.iteritems():
-
-    if plant.name in locations_dictionary.keys():
-        coords = locations_dictionary[plant.name]
-        plants_dictionary[idnr].location = pw.LocationObject(pw.NO_DATA_UNICODE, coords[0], coords[1])
-        plants_dictionary[idnr].coord_source = SOURCE_NAME
-    else:
+for idnr, plant in plants_dictionary.iteritems():
+    aux = aux_plant_info[plant.name]
+    lat, lon = aux['latitude'], aux['longitude']
+    try:
+        flat = float(lat)
+        flon = float(lon)
+    except:
         location_not_found += 1
         plants_dictionary[idnr].coord_source = pw.NO_DATA_UNICODE
-
-    if plant.name in commissioning_years_dictionary.keys():
-        plants_dictionary[idnr].commissioning_year = commissioning_years_dictionary[plant.name]
     else:
+        plant.location = pw.LocationObject(pw.NO_DATA_UNICODE, flat, flon)
+        plant.coord_source = SOURCE_NAME
+    try:
+        year = float(aux['commissioning_year'])
+    except:
         year_not_found += 1
+    else:
+        plant.commissioning_year = year
 
-"""
-print("Locations not found for these plants:")
-location_not_found.sort(key = lambda x:x.capacity, reverse=True)
-for plant in location_not_found:
-    if 'MOVIL' not in plant.name:
-        print(u"{0}, {1} MW".format(plant.name, plant.capacity))
-
-print("Commissioning year not found for these plants:")
-year_not_found.sort(key = lambda x:x.capacity, reverse=True)
-for plant in year_not_found:
-    print(u"{0}, {1} MW".format(plant.name, plant.capacity))
-"""
+print("Missing location for {0} plants; missing commissioning year for {1} plants.".format(location_not_found, year_not_found))
 
-print("Missing location for {0} plants; missing commissioning year for {1} plants.".format(location_not_found,year_not_found))
 
 # report on plants read from file
 print(u"...read {0} plants.".format(len(plants_dictionary)))
 
 # write database to csv format
-pw.write_csv_file(plants_dictionary, CSV_FILE_NAME)
+asdf = pw.write_csv_file(plants_dictionary, CSV_FILE_NAME)
 
 # save database
 pw.save_database(plants_dictionary, SAVE_CODE, SAVE_DIRECTORY)

build_databases/build_database_AUS.py

@@ -105,7 +105,7 @@
             owner = pw.format_string(plant.find("Electricity_Infrastructure:OWNER", ns).text)
         except:
             owner = pw.NO_DATA_UNICODE
-        fuel = pw.standardize_fuel(plant.find("Electricity_Infrastructure:PRIMARYFUELTYPE", ns).text,fuel_thesaurus)
+        primary_fuel = pw.standardize_fuel(plant.find("Electricity_Infrastructure:PRIMARYFUELTYPE", ns).text, fuel_thesaurus)
         try:
             capacity = plant.find("Electricity_Infrastructure:GENERATIONMW", ns).text
             capacity = float(capacity)
@@ -176,7 +176,7 @@
         new_plant = pw.PowerPlant(plant_idnr=idnr, plant_name=name, plant_owner=owner, 
             plant_country=COUNTRY_NAME,
             plant_location=new_location, plant_coord_source=geolocation_source,
-            plant_fuel=fuel, plant_capacity=capacity,
+            plant_primary_fuel=primary_fuel, plant_capacity=capacity,
             plant_generation=generation,
             plant_source=SOURCE_NAME, plant_cap_year=year_updated,
             plant_source_url=SOURCE_URL)

build_databases/build_database_BRA.py

@@ -123,7 +123,7 @@ def standardize_fuel_BRA(ceg_code):
     # TODO: handle only known edge case: Roca Grande (2535)
     ceg_code = cells[0].findall("font/a")[0].text.strip()
     plant_id = int(ceg_code[-11:-5])
-    fuel = standardize_fuel_BRA(ceg_code)
+    primary_fuel = standardize_fuel_BRA(ceg_code)
 
     # use CEG code to lookup coordinates
     ceg_code_short = ceg_code[0:16]
@@ -195,7 +195,7 @@ def standardize_fuel_BRA(ceg_code):
     new_location = pw.LocationObject(pw.NO_DATA_UNICODE, latitude, longitude)
     new_plant = pw.PowerPlant(plant_idnr=idnr, plant_name=name, plant_country=COUNTRY_NAME,
         plant_location=new_location, plant_coord_source=geolocation_source,
-        plant_fuel=fuel, plant_capacity=capacity,
+        plant_primary_fuel=primary_fuel, plant_capacity=capacity,
         plant_source=SOURCE_NAME, plant_source_url=SOURCE_URL, plant_cap_year=SOURCE_YEAR,
         plant_commissioning_year=op_year)
     plants_dictionary[idnr] = new_plant

build_databases/build_database_CAN.py

@@ -105,10 +105,10 @@
         owner = pw.NO_DATA_UNICODE
 
     try:
-        fuel = pw.standardize_fuel(row[fuel_col], fuel_thesaurus)
+        primary_fuel = pw.standardize_fuel(row[fuel_col], fuel_thesaurus, as_set=False)
     except:
         print(u"-Error: Can't read fuel for plant with name {0}".format(name))
-        fuel = pw.NO_DATA_SET
+        primary_fuel = pw.NO_DATA_UNICODE
 
     try:
         capacity = float(row[capacity_col])
@@ -144,7 +144,7 @@
     new_plant = pw.PowerPlant(plant_idnr=idnr, plant_name=name, plant_country=COUNTRY_NAME,
         plant_owner=owner, plant_cap_year=data_date,
         plant_location=new_location, plant_coord_source=geolocation_source,
-        plant_fuel=fuel, plant_capacity=capacity,
+        plant_primary_fuel=primary_fuel, plant_capacity=capacity,
         plant_source=SOURCE_NAME_1, plant_source_url=SOURCE_URL_1)
     plants_dictionary[idnr] = new_plant
 
@@ -201,10 +201,10 @@
         owner = pw.NO_DATA_UNICODE
 
     try:
-        fuel = pw.standardize_fuel(row[fuel_col], fuel_thesaurus)
+        primary_fuel = pw.standardize_fuel(row[fuel_col], fuel_thesaurus, as_set=False)
     except:
         print(u"-Error: Can't read fuel for plant with name {0}".format(name))
-        fuel = pw.NO_DATA_SET
+        primary_fuel = pw.NO_DATA_UNICODE
 
     try:
         latitude = float(row[latitude_col])
@@ -234,7 +234,7 @@
     new_plant = pw.PowerPlant(plant_idnr=idnr, plant_name=name, plant_country=COUNTRY_NAME,
         plant_owner=owner, plant_cap_year=data_date,
         plant_location=new_location, plant_coord_source=geolocation_source,
-        plant_fuel=fuel, plant_capacity=capacity,
+        plant_primary_fuel=primary_fuel, plant_capacity=capacity,
         plant_source=SOURCE_NAME_1, plant_source_url=SOURCE_URL_2)
     plants_dictionary[idnr] = new_plant
 
@@ -283,10 +283,10 @@
         except:
             print(u"-Error: Can't read capacity for plant {0}; value: {1}".format(name, row[capacity_col]))
         try:
-            fuel = pw.standardize_fuel(row[fuel_col], fuel_thesaurus)
+            primary_fuel = pw.standardize_fuel(row[fuel_col], fuel_thesaurus, as_set = False)
         except:
             print(u"-Error: Can't read fuel type for plant {0}.".format(name))
-            fuel = pw.NO_DATA_SET
+            primary_fuel = pw.NO_DATA_UNICODE
         try:
             latitude = float(row[latitude_col])
             longitude = float(row[longitude_col])
@@ -345,7 +345,7 @@
         new_location = pw.LocationObject(location, latitude, longitude)
         new_plant = pw.PowerPlant(plant_idnr=idnr_full, plant_name=name, plant_country=COUNTRY_NAME,
                     plant_location=new_location, plant_coord_source=geolocation_source_string,
-                    plant_fuel=fuel, plant_capacity=capacity,
+                    plant_primary_fuel=primary_fuel, plant_capacity=capacity,
                     plant_owner=owner, plant_generation=generation,
                     plant_source=source, plant_source_url=url,
                     plant_commissioning_year=commissioning_year)