Python Script

#Tested with python 2.7
#Last modified 12/01/2016

#Dependency for this code: genderize (https://github.com/SteelPangolin/genderize)
#Dependency for genderize: requests v1 (installable via pip)
#Genderize installation via pip : pip install git+https://github.com/SteelPangolin/genderize.git

#imports
import csv, re, sys
from genderize import Genderize
from copy import deepcopy

#logs console output, mostly for preliminary counts
#copied from http://stackoverflow.com/a/5916874
class Logger(object):
    def __init__(self, filename="Default.log"):
        self.terminal = sys.stdout
        self.log = open(filename, "a")

    def write(self, message):
        self.terminal.write(message)
        self.log.write(message)

#does some processing of name lists, getting rid of any extra white space and initials
def process_names(name_list):
    items_to_be_removed = []
    items_ending_in_commas = []
    #removing initals e.g. "j" or "j."
    for list_item in name_list:
        if re.search('^[a-zA-z]$|^[a-zA-z]\.$', list_item):
            items_to_be_removed.append(list_item)
    for removal_items in items_to_be_removed:
        name_list.remove(removal_items)
    #some names had extra whitespace or something
    while '' in name_list:
        name_list.remove('')
    #now we need to remove some extra items ending in commas that arise because of names formatted as "Smith, Ph.D, John"
    for list_item in name_list:
        if list_item[-1] == ',':
            items_ending_in_commas.append(list_item)
    for removal_items in items_ending_in_commas[1:]:
        if removal_items in name_list:
            name_list.remove(removal_items)
    return name_list

#opens csv, actually calls processing functions and dedups inside
def extract_pis(csv_string):
    #open csv, had some dialect problems so next few lines are voodoo
    f = open(csv_string, 'rU')
    csvf = csv.reader(f)
    with f as csvfile:
       dialect = csv.Sniffer().sniff(csvfile.read(1024))
       csvfile.seek(0)
       reader = csv.reader(csvfile, dialect)

       #go through list of names and parse out first name, removing duplicates
       full_first_pis=[]
       full_other_pis=[]
       for row in reader:
           if len(row) > 14 and row[0] <> "Project Title":
               full_first_pis.append(get_first_PI(row))
               if get_other_PIs(row) <> "None":
                   full_other_pis.extend(get_other_PIs(row))
        #dedup based on last, firm
       full_first_pis = dedup_PI_list(full_first_pis)
       full_other_pis = dedup_PI_list(full_other_pis)
    return full_first_pis, full_other_pis

#based on last, first name list dedups
def dedup_PI_list(first_last_list):
    deduped_list = []
    for item in first_last_list:
        if item not in deduped_list:
            deduped_list.append(item)
    return deduped_list

#gets primary PIs specified row and processes, returns one name -- should be last, first
def get_first_PI(row):
    name = process_names(row[12].split(" "))
    # find the item that ends in ","
    index_of_comma = 0
    for ii in xrange(len(name)):
        if name[ii][-1] == ",":
            index_of_comma = ii
            break
    if len(name) >= index_of_comma + 2:
        return [name[index_of_comma].capitalize(), name[index_of_comma + 1].capitalize()]
    elif name == ['Unavailable']:
        return ['Unavailable,', 'Unavailable']
    #people with only initials, no first name
    elif len(name) == 1:
        name.append('Unavailable')
        return name
    else:
        raise ValueError('Something is wrong with name parsing!')

#gets other PIs in specified row and processes, returns list of names -- should be last, first
def get_other_PIs(row):
    list_of_names = []
    if row[13] == "Not Applicable":
        return "None"
    else:
        names = process_names(row[13].split(";"))
        for name in names:
            name = process_names(name.split(" "))
            # find the item that ends in ","
            index_of_comma = 0
            for ii in xrange(len(name)):
                if name[ii][-1] == ",":
                    index_of_comma = ii
                    break
            if len(name) >= index_of_comma + 2:
                list_of_names.append([name[index_of_comma].capitalize(), name[index_of_comma + 1].capitalize()])
            elif name == ['Unavailable']:
                return ['Unavailable,', 'Unavailable']
            # people with only initials, no first name
            elif len(name) == 1:
                name.append('Unavailable')
                return name
            else:
                raise ValueError('Something is wrong with name parsing!')
    return list_of_names

#API calling function, has to separate list into batches of 10, output dictionary
def get_genderization(list_of_names):
    list_of_name_dicts = []
    number_of_requests = len(list_of_names)
    for kk in xrange(number_of_requests):
        current_name = list_of_names[kk]
        #You can comment this print out but it is helpful to see the progress of the genderization
        print current_name
        try:
            new_name = (Genderize().get([current_name[1]]))
            new_name[0][u'full_name'] = " ".join(current_name)
            list_of_name_dicts.extend(new_name)
        except:
            raise ValueError('Error in genderization')
            break


    return list_of_name_dicts

#takes in api output dictionary, returns none, saves csv with specified output name
def export(genderized_list_of_dicts, output_name):
    #exports a genderized list of dicts to csv file named output_name
    with open(output_name, 'w') as csvfile:
        fieldnames = [u'count', u'gender', u'name', u'probability', u'full_name',u'num_pi']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

        writer.writeheader()
        for jj in xrange(len(genderized_list_of_dicts)):
            writer.writerow(genderized_list_of_dicts[jj])


#takes in api output dictionary, returns none, prints out preliminary count
def count_out_dict(name_dict):
    #prints counts of different genders, treating a name 51% probability of being female as being .51 female and .49 male
    sum_of_male_prob = 0
    sum_of_female_prob = 0
    for mm in xrange(len(name_dict)):
        if name_dict[mm][u'gender'] == "female":
            sum_of_male_prob += 1-name_dict[mm][u'probability']
            sum_of_female_prob += name_dict[mm][u'probability']
        elif name_dict[mm][u'gender'] == "male":
            sum_of_male_prob += name_dict[mm][u'probability']
            sum_of_female_prob += 1-name_dict[mm][u'probability']

    print "Preliminary Count"
    print "Female",sum_of_female_prob
    print "Male",sum_of_male_prob
    print "Current ratio",float(sum_of_female_prob)/(sum_of_female_prob+sum_of_male_prob)


def runGenderizationScript(file, keyword, exportAll):
#file should be 'input/<filename>', keyword should be '<descriptive keyword>'
    log_file = "output/log_"+keyword+".txt"
    export_file = "output/names_"+keyword+".csv"

    sys.stdout = Logger(log_file)

    first_pis, other_pis = extract_pis(file)

    genderized_first_pis = get_genderization(first_pis)
    genderized_other_pis = get_genderization(other_pis)


    for entry in genderized_first_pis:
        entry[u'num_pi'] = 1
    for entry in genderized_other_pis:
        entry[u'num_pi'] = 2

    #check for first pi that is also other pi and remove
    cleaned_genderize_other_pis = []
    for pi_2 in genderized_other_pis:
        num_matches = 0
        for pi_1 in genderized_first_pis:
            if pi_1[u'full_name'] == pi_2[u'full_name']:
                num_matches +=1
        if num_matches == 0:
            cleaned_genderize_other_pis.append(pi_2)

    print cleaned_genderize_other_pis

    genderized_all_pis = deepcopy(genderized_first_pis)
    genderized_all_pis.extend(cleaned_genderize_other_pis)

    print("===================================================================================")
    print("PRELIMINARY COUNTS")
    print("Please add in names with no assigned gender by hand and calculate this ratio again!")
    print("===================================================================================")
    count_out_dict(genderized_all_pis)

    if exportAll:
        export(genderized_all_pis, export_file)

#TODO call the main function -- here is an example!
file ='input/SR_22Nov2016_113407_13085397.csv'
keyword = 'electrophysiology'

runGenderizationScript(file, keyword, 1)
Advertisements