update

Author: DominikBuchner

boldigger/additional_data.py

@@ -4,6 +4,7 @@
 from boldigger.boldblast_coi import slices
 from bs4 import BeautifulSoup as BSoup
 
+
 ## funtion to scan the BOLDresutls file and return a list
 ## of all unique Process ID's found, will also return the workbook
 ## ato later use it when saving the data
@@ -14,18 +15,29 @@ def retrieve_process_ids(xlsx_path):
     sheet = wb.active
 
     ## get type (coi or its/rbcl) and extract process ids
-    if sheet.cell(row = 1, column = 11).value == 'Process ID':
-        type = 'coi'
-        process_ids = [sheet.cell(row = row, column = 11).value for row in range(2, sheet.max_row + 1) if sheet.cell(row = row, column = 11).value != None and sheet.cell(row = row, column = 11).value != '']
-    elif sheet.cell(row = 1, column = 11).value == 'Similarity':
-        type = 'its_rbcl'
-        process_ids = [sheet.cell(row = row, column = 14).value for row in range(2, sheet.max_row + 1) if sheet.cell(row = row, column = 14).value != None and sheet.cell(row = row, column = 14).value != '']
+    if sheet.cell(row=1, column=11).value == "Process ID":
+        type = "coi"
+        process_ids = [
+            sheet.cell(row=row, column=11).value
+            for row in range(2, sheet.max_row + 1)
+            if sheet.cell(row=row, column=11).value != None
+            and sheet.cell(row=row, column=11).value != ""
+        ]
+    elif sheet.cell(row=1, column=11).value == "Similarity":
+        type = "its_rbcl"
+        process_ids = [
+            sheet.cell(row=row, column=14).value
+            for row in range(2, sheet.max_row + 1)
+            if sheet.cell(row=row, column=14).value != None
+            and sheet.cell(row=row, column=14).value != ""
+        ]
     else:
         raise ValueError
 
     ## remove duplicates and return
     return wb, list(unique_everseen(process_ids)), type
 
+
 ## function to loop through the process IDs and call the BOLD API  for additional data
 ## will return a dict in form of {Process ID: [BOLD Record ID, BIN, Sex, Life stage, Country, Identifier, Identification method, Institution storing]}
 def get_data(process_ids, processbar):
@@ -34,114 +46,159 @@ def get_data(process_ids, processbar):
 
     ## open a new html session
     with requests_html.HTMLSession() as session:
-
         ## initialize the final dict output
         process_id_dict = {}
 
         for id_pack in process_ids:
-
             ## request the data, retry if the request failes
             while True:
                 try:
-                    resp = session.get('http://www.boldsystems.org/index.php/API_Public/specimen?ids=' + '|'.join(id_pack))
+                    resp = session.get(
+                        "http://www.boldsystems.org/index.php/API_Public/specimen?ids="
+                        + "|".join(id_pack)
+                    )
                 except:
                     continue
                 break
 
             ## use beautifulsoup to find all records in the response
-            soup = BSoup(resp.text, 'html.parser')
-            records = soup.find_all('record')
+            soup = BSoup(resp.text, "xml")
+            records = soup.find_all("record")
 
             ## loop through records to extract interesting data
             for count in range(len(records)):
-
-                tags = ['processid', 'record_id', 'bin_uri', 'sex', 'lifestage', 'country', 'identification_provided_by', 'identification_method', 'institution_storing']
+                tags = [
+                    "processid",
+                    "record_id",
+                    "bin_uri",
+                    "sex",
+                    "lifestage",
+                    "country",
+                    "identification_provided_by",
+                    "identification_method",
+                    "institution_storing",
+                ]
 
                 ## search the data encapsuled by each tag
-                specimen_data = [records[count].find(tag) if records[count].find(tag) != None else '' for tag in tags]
+                specimen_data = [
+                    records[count].find(tag) if records[count].find(tag) != None else ""
+                    for tag in tags
+                ]
 
                 ## extract the texts from these tags
-                specimen_data = [datapoint.text if datapoint != '' else '' for datapoint in specimen_data]
+                specimen_data = [
+                    datapoint.text if datapoint != "" else ""
+                    for datapoint in specimen_data
+                ]
 
                 ## append a link to the specimen page to the results if user is interested in looking up even more data
-                specimen_data.append('http://www.boldsystems.org/index.php/MAS_DataRetrieval_OpenSpecimen?selectedrecordid=' + specimen_data[1])
+                specimen_data.append(
+                    "http://www.boldsystems.org/index.php/MAS_DataRetrieval_OpenSpecimen?selectedrecordid="
+                    + specimen_data[1]
+                )
 
                 ## put the data in a dictionary in form of processID: [listofinformation]
                 process_dict = {specimen_data[0]: specimen_data[1:]}
 
                 ## update the final output
                 process_id_dict.update(process_dict)
 
-            processbar.UpdateBar(round(100 / len(process_ids) * (process_ids.index(id_pack) + 1)))
+            processbar.UpdateBar(
+                round(100 / len(process_ids) * (process_ids.index(id_pack) + 1))
+            )
 
     return process_id_dict
 
+
 ## function to save the additional data to the input file
 def save_results(workbook, additional_data, xlsx_path, type):
-
     sheet = workbook.active
 
     ## headers to add to the resultfile
-    headers = ['Record ID', 'BOLD BIN', 'Sex', 'Life stage', 'Country', 'Identifier', 'Identification Method', 'Institution storing', 'Specimen page url']
+    headers = [
+        "Record ID",
+        "BOLD BIN",
+        "Sex",
+        "Life stage",
+        "Country",
+        "Identifier",
+        "Identification Method",
+        "Institution storing",
+        "Specimen page url",
+    ]
 
     ## first empty column depends on the type either 12 oder 15
-    start = 12 if type == 'coi' else 15
+    start = 12 if type == "coi" else 15
 
     ## write the header row
     for i in range(start, start + len(headers)):
-        sheet.cell(row = 1, column = i).value = headers[i - start]
+        sheet.cell(row=1, column=i).value = headers[i - start]
 
     ## add the additional data to the resultfile
     for row in range(2, sheet.max_row + 1):
         for column in range(start, start + len(headers)):
-            if sheet.cell(row = row, column = start - 1).value in additional_data.keys():
-                sheet.cell(row = row, column = column).value = additional_data[sheet.cell(row = row, column = start - 1).value][column - start]
+            if sheet.cell(row=row, column=start - 1).value in additional_data.keys():
+                sheet.cell(row=row, column=column).value = additional_data[
+                    sheet.cell(row=row, column=start - 1).value
+                ][column - start]
 
     workbook.save(xlsx_path)
 
+
 ## main function to controll the flow
 def main(xlsx_path):
-
     ## define a layout for the new window
     layout = [
-    [sg.Text('Download status', size = (15, 1), key = 'bar_des'), sg.ProgressBar(100, orientation = 'h', size = (20, 20), key = 'bar')],
-    [sg.Multiline(size = (50, 10), key = 'out', autoscroll = True)]
+        [
+            sg.Text("Download status", size=(15, 1), key="bar_des"),
+            sg.ProgressBar(100, orientation="h", size=(20, 20), key="bar"),
+        ],
+        [sg.Multiline(size=(50, 10), key="out", autoscroll=True)],
     ]
 
     ## run the download loop only once. After that only run event loop
-    window = sg.Window('Additional data download', layout)
-    bar = window['bar']
+    window = sg.Window("Additional data download", layout)
+    bar = window["bar"]
     ran = False
 
     while True:
-
-        event, values = window.read(timeout = 10)
+        event, values = window.read(timeout=10)
         ## start inner loop only once
-        window['out'].print("%s: Extracting process ID's." % datetime.datetime.now().strftime("%H:%M:%S"))
+        window["out"].print(
+            "%s: Extracting process ID's."
+            % datetime.datetime.now().strftime("%H:%M:%S")
+        )
         window.Refresh()
 
         if not ran:
-
             ## catch any wrong file formats here to avoid crash
             try:
                 wb, process_ids, type = retrieve_process_ids(xlsx_path)
             except ValueError:
-                window['out'].print('Wrong file format. Close to continue.')
+                window["out"].print("Wrong file format. Close to continue.")
 
             ## collect the data from BOLD
-            window['out'].print('%s: Downloading additional data.' % datetime.datetime.now().strftime("%H:%M:%S"))
+            window["out"].print(
+                "%s: Downloading additional data."
+                % datetime.datetime.now().strftime("%H:%M:%S")
+            )
             window.Refresh()
             additional_data = get_data(process_ids, bar)
 
             ## saving the data according to type
-            window['out'].print('%s: Saving the results.' % datetime.datetime.now().strftime("%H:%M:%S"))
+            window["out"].print(
+                "%s: Saving the results." % datetime.datetime.now().strftime("%H:%M:%S")
+            )
             window.Refresh()
             save_results(wb, additional_data, xlsx_path, type)
 
         ran = True
 
         ## get events (just closing) and values
-        window['out'].print('%s: Done. Close to continue.' % datetime.datetime.now().strftime("%H:%M:%S"))
+        window["out"].print(
+            "%s: Done. Close to continue."
+            % datetime.datetime.now().strftime("%H:%M:%S")
+        )
         window.Refresh()
         event, values = window.read()