# web scraping
from urllib.request import urlopen
from bs4 import BeautifulSoup

#Data and analysis
import numpy as np
import pandas as pd
import statsmodels.formula.api as sm
from sklearn import preprocessing, linear_model, metrics

# Plotting
import matplotlib.pyplot as plt
import seaborn as sns

%matplotlib inline
params = {'legend.fontsize': 'x-large',
          'figure.figsize': (10, 6),
         'axes.labelsize': 'x-large',
         'axes.titlesize': 20,
         'xtick.labelsize':'x-large',
         'ytick.labelsize':'x-large'}
plt.rcParams.update(params)

def count_results(query = None, location = "Boston", binary_level = 1):
    job_ids = pd.DataFrame()

    result_list = []

    # Find the number of results
    URL_for_count = "http://www.indeed.com/jobs?q=data+scientist+{}&jt=fulltime&l={}".format(query, location)
    soup_for_count = BeautifulSoup(urlopen(URL_for_count).read(), 'html.parser')

    results_number = soup_for_count.find("div", attrs = {"id": "searchCount"}).text
    number_of_results = int(results_number.split(sep = ' ')[-1].replace(',', ''))

    # Now loop through the pages. Viewing 100 results at a time means fewer page refreshes, which are a bottleneck.
    i = int(number_of_results/100)
    for page_number in range(i + 1):
        URL_for_results = "http://www.indeed.com/jobs?q=data+scientist+{}&jt=fulltime&l={}&limit=100&start={}".format(query, location, str(100 * page_number))
        soup_for_results = BeautifulSoup(urlopen(URL_for_results).read(), 'html.parser')
        results = soup_for_results.find_all('div', attrs={'data-tn-component': 'organicJob'})

        # Extract the ID for each job listing
        for x in results:
            job_id = x.find('h2', attrs={"class": "jobtitle"})['id']
            job_title = x.find('a', attrs={'data-tn-element': "jobTitle"}).text.strip().capitalize()
            job_link = "https://www.indeed.com" + x.find('h2', attrs={"class": "jobtitle"}).find('a')['href']
            result_list.append([job_id, job_title, job_link, binary_level])

        # Add the job ID numbers
        job_ids = job_ids.append(result_list)

    # Remove re-posted jobs
    job_ids.drop_duplicates(inplace = True)
    return job_ids

# String format: "keyword, keyword combination, keyword, ..."
def count_results_by_keywords(query_string = None):

    # Ends the function if given invalid inputs
    if query_string == None:
        return(print("No keyword entered."))

    # Format the keyword string in to URL query
    query = "%20OR%20".join(query_string.split(", "))
    query = query.replace(' ', '+')

    # Perform the search
    job_ids = count_results("%28{}%29".format(query))

    # Rename job_ids's columns
    job_ids.columns = ['id', 'job title', 'job link', '{}'.format(" OR ".join(query_string.split(", ")))]

    return (job_ids)


def count_results_by_salary(salary_range_divider = None, salary_floor = 20000):

    if salary_range_divider <= salary_floor:
        return(print("Enter a number larger than ${}.".format(salary_floor)))

    job_ids = pd.DataFrame()
    # Set dividing salaries
    divider_strings = ["+${}-{}".format(salary_floor, salary_range_divider), "+${}".format(salary_range_divider)]

    # Perform two searches, starting with the low-salary jobs
    for level, salary_criterion in enumerate(divider_strings):
        job_ids = job_ids.append(count_results(salary_criterion, binary_level=level))

    # Rename job_ids's columns
    job_ids.columns = ['id', 'job title', 'job link', 'salary over {}'.format(salary_range_divider)]

    return(job_ids)


def count_results_by_years_experience(years_experience = None):

    # Ends the function if given invalid inputs
    if years_experience == None or type(years_experience) != int:
        return(print("Enter an integer value."))

    # Format the keyword string in to URL query
    query = "{}+years+or+{}%2B+years".format(str(years_experience), str(years_experience))

    # Perform the search
    job_ids = count_results("%28{}%29".format(query), binary_level = years_experience)

    # Rename job_ids's columns
    job_ids.columns = ['id', 'job title', 'job link', 'years experience']

    return (job_ids)

phd_dataframe = count_results_by_keywords("PhD, ph.d")

bachelors_dataframe = count_results_by_keywords("Bachelors, BS, BA") # Indeed.com's search includes "Bachelor's", "B.S.", etc.

python_dataframe = count_results_by_keywords("Python")

# Use %22 ... %22 to ensure at least one of these keywords are present
management_dataframe = count_results_by_keywords("%22Manager%22, %22director%22")

# Rename the column to a readable format
management_dataframe.columns = ['id', 'job title', 'job link', 'Manager OR director']

startup_dataframe = count_results_by_keywords("Startup, start-up")

scientist_dataframe = count_results_by_keywords("Scientist")

machine_learning_dataframe = count_results_by_keywords("Machine learning")

lab_dataframe = count_results_by_keywords("Lab, laboratory")

# Use %22 ... %22 to search for the exact phrase "Software engineer"
software_engineer_dataframe = count_results_by_keywords("%22Software engineer%22")

# Rename the column to a readable format
software_engineer_dataframe.columns = ['id', 'job title', 'job link', 'Software engineer']

experience_results_dataframe = pd.DataFrame()

# Scan search results from a range of year requirements
for years in range(1+7):
    experience_results_dataframe = experience_results_dataframe.append(count_results_by_years_experience(years))

experience_results_dataframe['years experience'].astype(int).astype('category', ordered = True, copy = False)
years_dummies_dataframe = pd.get_dummies(experience_results_dataframe['years experience'], prefix = 'years experience', prefix_sep = ' ')
experience_dataframe = pd.concat(objs = [experience_results_dataframe.drop('years experience', axis = 1), years_dummies_dataframe], axis = 1)

salary_dataframe = count_results_by_salary(90000)

common_columns = ['id', 'job title', 'job link']
master_dataframe = phd_dataframe.merge(bachelors_dataframe, on = common_columns, how = 'outer')\
    .merge(python_dataframe, on = common_columns, how = 'outer')\
    .merge(startup_dataframe, on = common_columns, how = 'outer')\
    .merge(scientist_dataframe, on = common_columns, how = 'outer')\
    .merge(machine_learning_dataframe, on = common_columns, how = 'outer')\
    .merge(lab_dataframe, on = common_columns, how = 'outer')\
    .merge(software_engineer_dataframe, on = common_columns, how = 'outer')\
    .merge(management_dataframe, on = common_columns, how = 'outer')\
    .merge(experience_dataframe, on = common_columns, how = 'outer')\
    .merge(salary_dataframe, on = common_columns, how = 'outer')

# Convert non-id columns to integers and fill NaN values
data_conversion_mask = (master_dataframe.columns != 'id') & (master_dataframe.columns != 'job title') & (master_dataframe.columns != 'job link')
master_dataframe.ix[:, data_conversion_mask] = master_dataframe.ix[:, data_conversion_mask].fillna(value = 0).astype(int)

master_dataframe.head(3)

# Create a dataframe that keeps only the data features for analysis
data_dataframe_mask = (master_dataframe['Lab OR laboratory'] == 0) | (master_dataframe['Software engineer'] == 1)
data_dataframe = master_dataframe[data_dataframe_mask].drop(labels = common_columns, axis = 1).drop(labels = ['Lab OR laboratory', 'Software engineer'], axis = 1)
#data_dataframe.loc[:, 'years experience'] = sklearn.preprocessing.minmax_scale(data_dataframe['years experience'].astype(float), copy = False)

sm_model_dataframe = data_dataframe.copy()
sm_model_dataframe.columns = sm_model_dataframe.columns.str.replace(' ', '_').str.replace('.', '_').str.replace('-', '_')
sm_model_dataframe.columns

sm_model = sm.logit("salary_over_90000 ~ PhD_OR_ph_d + Bachelors_OR_BS_OR_BA + Python + Startup_OR_start_up + Scientist + Machine_learning + Manager_OR_director + years_experience_0 + years_experience_1 + years_experience_2 + years_experience_3 + years_experience_4 + years_experience_5 + years_experience_6 + years_experience_7", data=sm_model_dataframe).fit()
sm_model.summary()

import sklearn
X = data_dataframe.drop('salary over 90000', axis = 1)
y = data_dataframe['salary over 90000']
skl_logreg = sklearn.linear_model.LogisticRegressionCV(cv = 6)
skl_model = skl_logreg.fit(X, y)
predictions = skl_model.predict(X)

model_confusion_matrix = sklearn.metrics.confusion_matrix(y, predictions)

print("Average cross validation scores: {}".format(sklearn.cross_validation.cross_val_score(skl_logreg, X, y, scoring='roc_auc').mean()))
print("Confusion matrix:\n", model_confusion_matrix)
print("Model performance: \n", sklearn.metrics.classification_report(y, predictions))
print("Area under ROC curve for model: {}".format(sklearn.metrics.roc_auc_score(y, predictions)))

logreg_probabilites_dataframe = pd.DataFrame(skl_logreg.predict_proba(X), columns = ["Probability over threshold", "Probability under threshold"])
model_vs_data_dataframe = pd.concat(objs = [logreg_probabilites_dataframe, master_dataframe[['salary over 90000']]],axis=1)
model_vs_data_dataframe.head()

skl_logreg_model_coefficients = pd.DataFrame(data = skl_model.coef_, index = ['coefficient'], columns = data_dataframe.drop(labels = 'salary over 90000', axis = 1).columns).T

print("Ratio of jobs over salary threshold to under salary threshold: {}".format(round(data_dataframe['salary over 90000'].describe()['mean'],3)))

sns.countplot(data = data_dataframe, x = 'salary over 90000')
plt.title("Number of Jobs Under and Over Salary Threshold")
sns.plt.show()

master_dataframe[(master_dataframe['PhD OR ph.d'] == 1) & (master_dataframe['years experience 3'] == 1)][['job title', 'PhD OR ph.d', 'years experience 3']].head()

master_dataframe[master_dataframe['Lab OR laboratory'] == 1]['job title'].head()

print("Jobs analyzed: {}\nJob postings removed: {} ({}% change)\nRemaining postings: {}".format(master_dataframe.shape[0], master_dataframe.shape[0] - data_dataframe.shape[0], round(100 * (data_dataframe.shape[0] / master_dataframe.shape[0] - 1),1), data_dataframe.shape[0]))
print("\nKeywords\t\tFrequency\n{}".format(master_dataframe.drop('salary over 90000', axis = 1).describe().loc['mean'].sort_values(ascending = False)))

skl_logreg_model_coefficients.sort_values(by = 'coefficient', ascending = False)

plt.figure(figsize=(10,6))
sns.barplot(data = skl_logreg_model_coefficients.sort_values(by = 'coefficient', ascending = False).T, orient='h')
plt.title("Influence of Features on Predicting Job Salary")
sns.plt.show()