Useful python scripts for daily uses

Welcome to my collection of Python scripts and code snippets for everyday use. This document contains a variety of scripts and code snippets that can be used to automate repetitive tasks, process data, and perform various other operations. Whether you are a beginner or an experienced Python programmer, this collection of scripts and code snippets will provide you with useful tools and shortcuts for your everyday work.

In this document, you will find code snippets and scripts for tasks such as:

• Data processing
• File manipulation
• Web scraping
• Automating tasks
• And more!

This document is organized into sections for easy navigation, so you can quickly find the script or code snippet that you need. Many of them were collected from stackoverflow.

On 2022.03.31, I re-organize all the scripts into Python 3.x version and post again.
On 2023.02.12, I edited the whole content with help from ChatGPT

1.Data processing

#1. Find the duplicated values in one column
from collections import Counter   
print ([item for item, count in Counter(df["Sample"].values).items() if count >1] )

#2. Build new dataframe based on the user-defined column names
### N: length
def new_df(N, LIST):
    df = pd.DataFrame(np.nan, index=np.arange(0,N,1), columns=['A'])
    for i,t in enumerate(LIST):
        df[LIST[i]] = np.nan
        df = df.drop(['A'],1)  

#3. Change the integer to N digits 
## method 1
# Method 1
df.MONTH.apply("{0:0=2d}".format)
# Method 2
"%02d" & (string)

#4. Replace the values in specific columns
pd.options.mode.chained_assignment = None
df.loc[condition function, 'column'] = xxx  

#5. Find the index of the nearest value to a pre-defined input
def find_nearest(array,value):
    idx = (np.abs(array-value)).argmin()
    return array[idx]   

#6. Test a value if it is NaN:      
def isNaN(num):
    return num != num

#7. Test dataframe if it is empty
df.empty==True

#8. Zip two pandas columns into 2-d array
from itertools import product        
c = list(product(df.lon.values, df.lat.values))       

#9. Manipulate the string
## Add an element
list.insert(0, 'A')

## Delete an element by index
list.pop(2)

## Delete an element by its value
list.remove('A')

#10. Assign value to specific location of a dataframe
## Method 1 - the fastest
df.set_value('row index', 'column name', value)

## Method 2 - moderate speed
df['column name']['row index'] = value

## Method 3 - the slowest
df.at('column name', 'row index') = value

#11. Group the dataframe with user-defined range setting
bins = [0,50, 60, 70, 80, 90,100]
group_names = ['<50', '50-60', '60-70', '70-80','80-90','>90']
df['categories'] = pd.cut(df['RH'], bins, labels=group_names)
## If we want to plot them group-name wised
fig = plt.figure()
for i in range(0,len(categories.unique())-1,1):
    ddf = df[df['categories'] == categories.unique()[i]]
    plt.scatter(ddf['PM2.5'], ddf['VISIBILITY'], alpha = 0.45, label ='RH: '+ roup_names[i])    
    popt, pcov = curve_fit(func, ddf["PM2.5"], ddf["VISIBILITY"])        
    xdata = np.arange(0, 300, 4)
    plt.plot(xdata, func(xdata, *popt),  zorder =3)
plt.legend(ncol = 2)    
plt.show()    

#12. Transform the list to string
import ast
x = u'[ "A","B","C","D"]'
x = ast.literal_eval(x)
print x  # Output: ABCD

#13. Subset of pandas dataframe by datetime range
mask = (met['Date'] < '2016-03-01')|(met['Date'] >= '2016-12-01')
met_sub = met.locmask

#14. Sort one column by values of another column (ranking species by their concentrations)
SE_COL = ['Cr', 'Co', 'Ni', 'Cu', 'Zn', 'As', 'Se', 'Mo', 'Cd', 'Pb', 'V', 'Mg', 'Ca', 'K', 'Ti', 'Mn', 'Fe', 'Ba', 'Sr']
SE_COL = [SE_COL[t] for t in np.argsort(np.array([vals.dropna().mean() for col, vals in pm25_ef.items()]))]

#15. Drop np.nan and then find the 10th percentile
v = df['value'].notna()
ra_ = df.loc[v, 'value'].values
se_index = ra_.argsort().argsort()[:int(0.1len(df))]
sorted(ra_)[int(0.1len(ra_))]

#16. Rename the columns of dataframe
df.rename(columns={'oldName1': 'newName1', 'oldName2': 'newName2'}, inplace=True)

#17. Delete specific columns
columns = ['Col1', 'Col2', ...]
df.drop(columns=columns, inplace=True, axis = 1)

#18. Add new columns
df[new_column_name] = pd.Series(data, index=df.index)

#19. Delete the rows with np.nan
## Method 1
df = df[~np.isfinite('col1')]
## Method 2
df.dropna(subset=['col1'], inplace=True)

#20. Sort the values from the biggest
var_exp = sorted(eigen_vals, reverse=True)

#21. Saving a CSV file containing Chinese characters
##To save a CSV file containing Chinese characters, you need to make sure that the file is encoded in UTF-8 format.
import sys
reload(sys)
sys.setdefaultencoding('utf-8')    
df.to_csv('file.csv', encoding='utf-8-sig', index=False)


#22. Adding a day to a datetime object
## To add one day to a datetime object, you can use the pd.to_timedelta() function. For example:
date = pd.to_datetime('2017-11-01')
new_date = date + pd.to_timedelta(1, 'D')

#23. Getting the keys and values of a dictionary
keys = [key for key, value in dict_.items()]
values = [value for key, value in dict_.items()]

#24. Binning data with natural break
from pysal.esda.mapclassify import Natural_Breaks as nb
breaks = nb(df.value,k=5)
class_ = pd.DataFrame({'class': breaks.yb}, index=df[df['value'].notnull()].index)
df = df.join(class_)
df.class.fillna(-1, inplace=True)

#25. Printing a pandas DataFrame in Markdown format
from tabulate import tabulate
print(tabulate(response_df, tablefmt="pipe", headers="keys", showindex=False))

#26. saving the 2-d array in txt and read it
np.savetxt("mask_PM_0.1x0.1.txt",mask)
mask = np.loadtxt('./mask_PM_0.1x0.1.txt')

#27. np.array dropna for the NaN of ratios 
t = np.array([(a/b) for a, b in zip(dyn_w[spec].values,dyn_s[spec].values)])
t = t[~np.isnan(t)]

# 28. How to select the data points within specific area
### 长三角
yrd_area = fiona.open("./data/shp/YRD/yrd_b.shp")
pol = yrd_area.next()
from shapely.geometry import shape
geom = shape(pol['geometry'])
poly_data = pol["geometry"]["coordinates"][0][0]#[0:12]
poly = Polygon(poly_data)
yrd_mask = np.array([poly.contains(Point(x,y)) for x,y in zip(data_city['lon'],data_city['lat'],)])
data_yrd = data_city[yrd_mask]
data_yrd = data_yrd.reset_index()

# 29. 存储中文xls报错问题处理
df = pd.read_csv('./xxx.txt', delimiter = "\t",encoding='utf-8')
with pd.ExcelWriter('xxx.xls',options={'encoding':'utf-8'}) as writer:  #
    df.to_excel(writer, sheet_name=u'sheet1')

# 30. split the list by specfic string (e.g., quotes)
[s for s in l.split('"') if s.strip() != ''][1]

# 31. replace specific values by condition 
df.loc[df['COLA']!='XXX','COLB']=yyy

# 32. Select subset of netcdf file by specific attributes
import netCDF4 as nc
val_ = ['HGT_M', 'XLONG_C','XLAT_C']
with netCDF4.Dataset("geo_em.d01.nc") as src, netCDF4.Dataset("out.nc", "w") as dst:
    # copy global attributes all at once via dictionary
    dst.setncatts(src.__dict__)
    # copy dimensions
    for name, dimension in src.dimensions.items():
        dst.createDimension(
            name, (len(dimension) if not dimension.isunlimited() else None))
    # copy all file data except for the excluded
    for name, variable in src.variables.items():
        if name in val_:
            x = dst.createVariable(name, variable.datatype, variable.dimensions)
            dst[name][:] = src[name][:]
            # copy variable attributes all at once via dictionary
            dst[name].setncatts(src[name].__dict__)

# 33. combining the lists
code_ = []
for pr in pc_code:
    code_ = list(itertools.chain.from_iterable([code_,[t['code'] for t in pr['children']]]))    

# 34. 2-D数据插值
#向新网格进行插值
from scipy import interpolate
hfunc = interpolate.interp2d(lon_dust_sub,lat_dust_sub,PM_dust)
new_dust = np.zeros(len(lat_anthr_sub)*len(lon_anthr_sub))
t =  0 
for i in range(0,len(lat_anthr_sub),1):
    for j in range(0,len(lon_anthr_sub),1):
        new_dust[t] = hfunc(lon_anthr_sub[j],lat_anthr_sub[i])
        t+=1
dust_int = new_dust.reshape(len(lat_anthr_sub),len(lon_anthr_sub))

# 35. 排序获得其序列index
sorted(range(len(s)), key=lambda k: s[k])

# 39. Sort时注意排序
sorter = source_
df_contr_cr.source = df_contr_cr.source.astype("category")
df_contr_cr.source.cat.set_categories(sorter, inplace=True)
df_contr_cr = df_contr_cr.sort_values(["source"]) 

df['categories'].value_counts().reindex(group_names)

# 39. 分钟数据转化为小时数据
df2_re = df2.groupby(np.arange(len(df2))//60.0).mean()

# 40. 有效位数输出
to_clipboard(float_format = '%.4f')

# 41.自然基金报告下载
import os
for i in range(0,100,1):
    os.system("wget http://output.nsfc.gov.cn/report/41/41205122_%i.png" %i)    
import os
import natsort
from fpdf import FPDF

files = os.listdir('./')
files = natsort.natsorted(files, reverse = False)
files = files[0:]
pdf = FPDF()

for image in files[0:]: 
    if image[-3:]=='png': 
        pdf.add_page()  
        pdf.image(image,0,0,210,297)  
pdf.output("../hh.pdf", "F")   

# 43. 缺失值监测
pd.isnull()
dataframe[xxx].values.isnull()

# 44. If Else in list comprehension
[x for x in list(df.columns[2:]) if x not in ['重庆','山西','西藏']]

# 45. Two variables in list comprehension (double iteration)

# 46. assign two value to pandas columns with if condition.
contr_df['type'] = contr_df['sample'].apply(lambda x:10 if x[0:7] =="IITK-20" else "x")

# 46. txt file reading with multiple delimiter and skiplines
df = pd.read_csv('./data/210717_Lucknow_Day1.txt',delimiter='\t',skiprows=[i for i in range(0,10)])

# 47. add multiple lists as the columns of new dataframe
dpara = pd.DataFrame(list(zip(t_series,rn_series,minErr1ion)),columns=['t_series','rn_series','minErr1ion'])

# 48. replace all the quotation mark in the column
species_name  =  [str(c.rstrip().split("\t")[1].replace('"', '')) for c in species_info1]

# 49. Kruskal-Wallis test
from scipy import stats
stats.kruskal(dd['Glass vial'],dd['Plastical vial'],dd['Dilution water'])

# 50. list substraction
list(set( list(dafr_no3.columns)) - set(spike_cols)-set(['time'])

# 51. Assign 2-d matrix into pandas dataframe with column name
     dafr_org = pd.DataFrame(org_, columns=df_org['Parameter'].attrs['ExactMassText'].astype(str))

# 52. divide string based on chemical formula     

import re
c = 'C2H3O'
splitted = list(re.split("([A-Z][a-z]*)",c))
keyss = list(filter(lambda a: a[0].isupper() if a else False, splitted))
values = list(filter(lambda a: a[0].isdigit() if a else False, splitted))
spec_dict = dict(zip(keyss,values)) 

# 53. check string type in columns
na_em_power.applymap(type)==str

# 54. Sorting files by specific strings witin the filename
def last_4chars(x):
    return(x[-25:])

sorted(imgs, key = last_4chars)   

# 55. split string by more than one delimiters
import re
re.split('; |, ', string_to_split)

# 56. Pandas rename multiple columns
df_area.rename(columns=dict(zip(df_area.columns[:], mz_list)),inplace=True)

# 57. Pandas check whether the whole column is np.nan
for c in mz_list[0:]:
    if df_area[c].isnull().all():
        print("All values in the column 'B' are NaN")

# 58. Rename part of column names by condition
def renaming_fun(a):
    if "_unc_new_toc_scaled" in a:
        return a.split('_')[0] +"_unc" # or None
    return a
auto_df_fa.columns = [renaming_fun(col) for col in auto_df_fa.columns]


# 58. Append 2-d array into 3-d array
### https://stackoverflow.com/questions/43363641/building-a-3d-array-from-a-number-of-2d-arrays-with-numpy
OC_emissions_daily  = np.empty((1,720, 1440), int)
daily_em_array      = month_OC_emissions * daily_fraction
OC_emissions_daily  = np.vstack([OC_emissions_daily, daily_em_array[None]])

# 59. processing pandas multiple columns 
df[df.columns[2:]] = df[df.columns[2:]].multiply(spi_conc/RIE/c, axis="index")

# 60. Lambda function
### Apply function NumPy.square() to square the values of two rows 'A'and'B
df2 = df.apply(lambda x: np.square(x) if x.name in ['A','B'] else x)

# 61. Saving n-d array and reload via pickle method
import numpy as np
import pickle

# 62. create a large numpy array
data = np.random.rand(1000, 1000, 100)

# 63. save the numpy array to a file using pickle
with open('data.pkl', 'wb') as f:
    pickle.dump(data, f)

# 64. load the numpy array from the file
with open('data.pkl', 'rb') as f:
    loaded_data = pickle.load(f)

# 65. check if the original and loaded numpy arrays are the same
print(np.array_equal(data, loaded_data))

# 66. Daily average
df.set_index('datetime',inplace=True)
df = df.rename(columns=new_column_names)
daily_ave = df.resample('D').mean()


# 67. Create a Pandas Excel writer using 'xlsxwriter' engine
with pd.ExcelWriter('my_excel_file.xlsx', engine='xlsxwriter') as writer:
    # Write dataframes df1 and df2 to two separate sheets in the same Excel file
    df1.to_excel(writer, sheet_name='Sheet1')
    df2.to_excel(writer, sheet_name='Sheet2')
     
     
# 68. Simple way to classify chemical formula
def classify_chemicals(formula): 
     if pd.isnull(formula): 
         return np.nan 
     else: 
         elements = ['C', 'H', 'O', 'N', 'Na'\] 
         components = re.findall('\[A-Z\]\[a-z\]*', formula) 
         # Split formula into components 
         category = ''.join(\[element for element in elements if element in components]) 
     return category 
ds['Ion-rule1-class'] = ds['ion-rule1'].apply(classify_chemicals)

2. Data statistics

#1. Curve fitting (y = ax + b)
from scipy import stats
import numpy as np
slope, intercept, r_value, p_value, std_err = stats.linregress(x,y)
##1.1 remove the nan value first!
mask = ~np.isnan(x_) & ~np.isnan(y_)
slope, intercept, r_value, p_value, std_err = stats.linregress(x_[mask], y_[mask])

#2. Fitting by user-defined function 
from scipy.optimize import curve_fit        
def func(x, a,b):
    return ax**2+bx#+0.55
def get_r(f,x,y,popt):
    residuals = y- f(x, *popt)
    ss_res = np.sum(residuals**2)    
    ss_tot = np.sum((y-np.mean(y))**2)    
    r_squared = 1 - (ss_res / ss_tot)
    return r_squared
popt, pcov = curve_fit(func, df['x'], df['y']) 
print "## f(x) = ax*2 +bx ##"
print "## a##"
print popt
print "## r-square ##"
print get_r(func,df['x'], df['y'], popt )

#3. pivot table 
df.groupby(['site']).get_group(df.site.unique()[i])['value'].mean()

#4. Summarize the frequency
clu_d['Cluster'].value_counts().to_frame()
clu_d['Cluster'].value_counts().index.tolist()  

#5. one-way ANOVA test
# 如何统计 seasonal anova difference
spec_ = ['crustal','trace','OM',"EC",'SO42-','NO3-','Cl-','NH4+','n_K',]
val_annual,val_sp,val_su,val_au,val_wi = [],[],[],[],[]
for t in sp_pie:
    val_sp=df[df.season=='Spring'][t].dropna().values
    val_su=df[df.season=='Summer'][t].dropna().values
    val_au=df[df.season=='Autumn'][t].dropna().values
    val_wi=df[df.season=='Winter'][t].dropna().values
    print t
    print stats.f_oneway(val_sp,val_su, val_au,val_wi)

# pearson correlation 
mask = ~np.isnan(df['Ma_re'])&~np.isnan(df['PM25'])
stats.pearsonr(df['Ma_re'][mask],df['PM25'][mask])


# 6. D-N averaged
sample_id = df_op[df_op['Sample ID'].str[-1] !='C']['Sample ID'].values
#IITK-2018-02-12-N #IITK-2018-01-21-N
sample_id = [c for c in sample_id if c not in ['IITK-2018-02-12-N' ,'IITK-2018-01-21-N',]]

df_op_c = df_op[~(df_op['Sample ID'].isin(sample_id))]
df_op_c  = df_op_c[df_op_c['Sample ID']!='IITK-2018-11-01-C']

df_op_dn = pd.DataFrame()
df_op_dn['Sample ID'] = [c[:-2]+'-C' for c in sample_id]
df_op_dn['Date'] =  [c[5:-2] for c in sample_id]
for co in df_op.columns[2:]:
    val_ = []
    for c in sample_id:
        val_.append(df_op[df_op['Sample ID'].str[:-2] == c[:-2]][co].mean())
    df_op_dn[co] = val_
    
df_op_new = pd.concat([df_op_dn,df_op_c]).reset_index(drop=True)    

## OM:OC

from pyvalem.formula import Formula

def cal_omoc(ion):
    f         = Formula(ion)
    atom_dict = f.atom_stoich
    n_c       = atom_dict['C']
    omoc_ratio = f.mass/(n_c*12.0)
    
    return omoc_ratio
pro_pr['OM:OC'] = pro_pr['Ion'].apply(lambda ion: cal_omoc(ion))

3. Data visualization

#1. Temperature symbol
ax.set_ylabel(u'Temperature (\N{DEGREE SIGN}C)', fontweight='bold')

#2. Match the length of colorbar with the figure
from mpl_toolkits.axes_grid1 import make_axes_locatable
divider = make_axes_locatable(ax1)
cax = divider.append_axes("right", size="5%", pad=0.1)
cbar = plt.colorbar(cf, cax=cax)    
cbarlabel = "xxxx"
cbar.ax.set_xlabel(cbarlabel,size = 8,labelpad=-35)

#3. User-defined colorbar location and ticklabel editting
cbaxes = fig.add_axes([0.83, 0.125, 0.1, 0.01]) 
cbar = plt.colorbar(ss,cax=cbaxes,orientation='horizontal')
loc_ = np.array([5,10,15,20,25])
cbar.set_ticks(loc_)
cbar.set_ticklabels(loc_)
## two methods for ticklabel fontsize
(1) cbar.ax.tick_params(labelsize=8)
(2) cbar.ax.set_xticklabels(cbar.ax.get_xticklabels(), fontsize=8)
cbar.ax.set_xlabel(r'$\mathregular{OC/EC}$',fontsize = 8,labelpad = -28)

#4. Chinese setting
from matplotlib.font_manager import FontProperties 
mpl.rcParams['font.sans-serif'] = ['Microsoft YaHei']# 微软雅黑的中英文混排效果较好
plt.rcParams['axes.unicode_minus'] = False  

#5. User-defined axes setting
def stylize_axes(ax):
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.xaxis.set_tick_params(top='off', direction='out', width=1)
    ax.yaxis.set_tick_params(right='off', direction='out', width=1)

#6. Plotting the first-order regression line
## (x,y) is the original datasets
from scipy import stats
slope, intercept, r_value, p_value, slope_std_error = stats.linregress(x, y)
xx = np.arange(x.min(),x,max(),0.01)
yy = slope*xx+intercept
clabel = # make the font itatilc style
plt.plot(xx,yy,color ='k', lw =1.2, linestyle = "-", label = clabel)

#7. Setting the two subplots shared the same aspect ratio
## Noted that when one subplot using basemap or cartopy (geo-based coordinate), and the other subplot is in basic matplotlib style, there aspect ratio would not be the same. 
def get_aspect(ax):
    xlim = ax.get_xlim()
    ylim = ax.get_ylim()
    aspect_ratio = abs((ylim[0]-ylim[1]) / (xlim[0]-xlim[1]))
    return aspect_ratio
fig = plt.figure(figsize=(12,8))
ax1=plt.subplot(121, projection=ccrs.PlateCarree())
ax2=plt.subplot(122)
ax2.set_aspect(get_aspect(ax1) / get_aspect(ax2))

#8. Fake legend markers
#参考 https://jakevdp.github.io/PythonDataScienceHandbook/04.06-customizing-legends.html
for area in [100, 300, 500]:
    plt.scatter([], [], c='k', alpha=0.3, s=area,
                label=str(area) + ' km$^2$')
plt.legend(scatterpoints=1, frameon=False, labelspacing=1, title='City Area')

#9. Remove the plotting items and retain the label

#10. Remove the tick lines while retain the ticklabels
ax.tick_params(axis=u'both', which=u'both',length=0)

#11. plotting two subplots with different size
fig = plt.figure(figsize=(7,4))
##definitions for the axes
left, width = 0.07, 0.6
bottom, height = 0.1, .8
bottom_h = left_h = left+width+0.05
rect_cones = [left, bottom, width, height]
rect_box = [left_h, bottom, 0.5, height]
ax1 = plt.axes(rect_cones,projection=ccrs.PlateCarree())
ax2 = plt.axes(rect_box)
ax2.set_aspect(get_aspect(ax1) / get_aspect(ax2))

#12. fake circle legend
def plot_legend(title):
    s = [1,3,5,7,9]
    c = plt.cm.binary(np.arange(5)/5.0)
    labels =['A',"B",'C',"D","E"]
    cir1 =  plt.scatter([1], [2], c=c[0], alpha=0.8, s=s[0],label=r'<0.5')
    cir2 =  plt.scatter([1], [2], c=c[1], alpha=0.8, s=s[1],label=labels[1])
    cir3 =  plt.scatter([1], [2], c=c[2], alpha=0.8, s=s[2],label=labels[2])
    cir4 =  plt.scatter([1], [2], c=c[3], alpha=0.8, s=s[3],label=labels[3])
    cir5 =  plt.scatter([1], [2], c=c[4], alpha=0.8, s=s[4],label=labels[4])    
    leg = plt.legend([cir1,cir2,cir3,cir4,cir5],labels,scatterpoints = 1,  \
                     frameon=False, labelspacing=0.9, ncol =2,
                     fontsize=8,title=title , loc = [0.05,0.1])   

#13. seperated colors from colormap
c_list = plt.cm.rainbow(np.arange(6)/6.0)

# 14. set twin axis with different color
axu = ax.twinx()
axu.spines["right"].set_position(("axes", 1.25))
so_,=plt.plot(xxx)
axu.yaxis.label.set_color(so_.get_color())
axu.spines['right'].set_color(so_.get_color())
axu.spines["right"].set_edgecolor(so_.get_color())
axu.tick_params(axis='y', colors=so_.get_color())

# 15. add a rectangle
from matplotlib.patches import Rectangle
import matplotlib.patches as mpatches
rec = mpatches.Rectangle((position[i]-
                    width/2.0,bot_[i]),width,hei_[i],linewidth=1,edgecolor='b',facecolor='none',zorder=12)
    ax.add_patch(rec)

# 16. output the color by pre-defined alpha
def make_rgb_transparent(rgb, bg_rgb, alpha):
    return [alpha * c1 + (1 - alpha) * c2
            for (c1, c2) in zip(rgb, bg_rgb)]
from matplotlib import colors
import matplotlib.pyplot as plt

alpha = 0.5

kwargs = dict(edgecolors='none', s=3900, marker='s')
for i, color in enumerate(['red', 'blue', 'green']):
    rgb = colors.colorConverter.to_rgb(color)
    rgb_new = make_rgb_transparent(rgb, (1, 1, 1), alpha)
    print(color, rgb, rgb_new)
    plt.scatter([i], [0], color=color, **kwargs)
    plt.scatter([i], [1], color=color, alpha=alpha, **kwargs)
    plt.scatter([i], [2], color=rgb_new, **kwargs)

# 17. terrain map
def truncate_colormap(cmap, minval=0.0, maxval=1.0, n=100):
    new_cmap = colors.LinearSegmentedColormap.from_list(
    'trunc({n},{a:.2f},{b:.2f})'.format(n=cmap.name, a=minval, b=maxval),
    cmap(np.linspace(minval, maxval, n)))
    return new_cmap
cmap = plt.get_cmap('terrain')
terrain_cmap = truncate_colormap(cmap, 0.15, 0.9)  

# 18. 不等距subplots
import matplotlib.gridspec as gridspec
fig = plt.figure(figsize=(9,6))
gs = gridspec.GridSpec(46,1)
row_xx = 23
# pm25_pm10
ax1 = plt.subplot(gs[0:row_xx, 0])

# 19. two axis one legend
# ask matplotlib for the plotted objects and their labels
lines, labels = ax.get_legend_handles_labels()
lines2, labels2 = ax2.get_legend_handles_labels()
ax2.legend(lines + lines2, labels + labels2, loc=0)

# 20. Cartopy longitude
## 方法1
ax.set_xticks([0, 60, 120, 180, 240, 300, 360], crs=ccrs.PlateCarree())d
ax.set_yticks([-90, -60, -30, 0, 30, 60, 90], crs=ccrs.PlateCarree())
lon_formatter = LongitudeFormatter(zero_direction_label=True)
lat_formatter = LatitudeFormatter()
ax.xaxis.set_major_formatter(lon_formatter)
ax.yaxis.set_major_formatter(lat_formatter)
## 方法2
ax.set_xticks(np.arange(extent[0]+2, extent[1]+2, 15), crs=ccrs.PlateCarree())
ax.set_yticks(np.arange(extent[2]+3, extent[3], 15), crs=ccrs.PlateCarree())
ax.set_xticklabels([r'$\mathrm{75^o E}$',r'$\mathrm{90^o E}$',r'$\mathrm{105^o E}$',\
                    r'$\mathrm{120^o E}$',r'$\mathrm{135^o E}$',])
ax.set_yticklabels([r'$\mathrm{20^o N}$',r'$\mathrm{35^o N}$',r'$\mathrm{50^o N}$'])

# 21 'GeoAxesSubplot' object has no attribute '_hold'
from matplotlib.axes import Axes
from cartopy.mpl.geoaxes import GeoAxes
GeoAxes._pcolormesh_patched = Axes.pcolormesh

# 22.中英文混排
# # -*- coding: utf-8 -*-
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties

song_ti = FontProperties(fname=r'/Library/Fonts/Songti.ttc', size=20)
times_new_roman = FontProperties(fname=r'/Library/Fonts/Arial Black.ttf', size=15)
ax = plt.gca()
ax.set_title(u'能量随时间的变化', fontproperties=song_ti)
ax.set_xlabel('Time (s)', fontproperties=times_new_roman)
ax.set_ylabel('Energy (J)', fontproperties=times_new_roman)
plt.show()

# 23. no-legend
(label='_nolegend_')

# 25. plot with grey background
ax.set_facecolor("#F5F5F5")

# 26. Midpoint-normalized
import matplotlib.colors as colors
#https://stackoverflow.com/questions/25500541/matplotlib-bwr-colormap-always-centered-on-zero
class MidpointNormalize(colors.Normalize):
    def __init__(self, vmin=None, vmax=None, midpoint=None, clip=False):
        self.midpoint = midpoint
        colors.Normalize.__init__(self, vmin, vmax, clip)

    def __call__(self, value, clip=None):
        # I'm ignoring masked values and all kinds of edge cases to make a
        # simple example...
        x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1]
        return np.ma.masked_array(np.interp(value, x, y))


# 27. 月份colorbar
cbaxes = fig.add_axes([0.29, 0.27, 0.18, 0.02]) 
cbar = plt.colorbar(ss,cax=cbaxes, orientation='horizontal')
n = 6
st_po = []
for i in range(0,n,1):
    st_po.append(np.array_split(pd.to_datetime(sorted(date_point)),n)[i].values[0])
cb_ticks = [float(c) for c in st_po]
cbar.ax.set_xticklabels(pd.to_datetime(cb_ticks).strftime('%b')) #B     
cbar.ax.tick_params(labelsize=8.5)  

# 28. nclcmaps
import nclcmaps
nclcmaps.cmaps("precip2_17lev")

# 29. 创建新的colormap
 cm = LinearSegmentedColormap.from_list('test',plt.cm.BuPu(np.arange(6)/6.0)[1:], N=5)


 # 30. 最好的terrain map
http://chris35wills.github.io/discrete_colourbar/

# 31. 对数坐标系统显示实际数值
from matplotlib.ticker import StrMethodFormatter, NullFormatter
ax.yaxis.set_major_formatter(StrMethodFormatter('{x:.3f}'))
ax.yaxis.set_minor_formatter(NullFormatter())  

# 32. loop figure for subplot
fig, axs = plt.subplots(2,4, figsize=(15, 6), facecolor='w', edgecolor='k')#subplot_kw={'projection': ccrs.PlateCarree()}
fig.subplots_adjust(hspace = .5, wspace=.001)
axs = axs.ravel()

for i in range(8):

    axs[i].contourf(np.random.rand(10,10),5,cmap=plt.cm.Oranges)
    axs[i].set_title(str(250+i))

# patch_box  
from matplotlib.collections import PatchCollection
from matplotlib.patches import Rectangle
def PATCH_BOX(ax,pos,wid):
    for x in pos[::2]:
        width = wid/2.0
        facecolor = "#F0F0F0" 
        rect = Rectangle((x-width, 0.00), width*4.0,1000000,facecolor=facecolor,linewidth = 0)#           clip_on=False,
        ax.add_patch(rect)     

# 33. cartopy extent 报错问题
https://github.com/SciTools/cartopy/issues/837
pip uninstall shapely && pip install --no-binary :all: shapely

# 34. shapely的向量化总是报错
# https://github.com/Toblerity/Shapely/issues/810
pip uninstall shapely & pip install shapely --no-binary shapely==1.7a2

 # 35. scatter图的外环
linewidths=1

# 36. discrete color from continuous colormap
from matplotlib import cm
cs=cm.Set2(np.arange(4)/4.)

# 36. how to plot circle on unequal axes 
ax.plot(0.3,-0.1 , 'ro', fillstyle='full', markersize=5, transform=ax.transAxes,clip_on =False)

# 37. Plotting multi-polygons with pre-defined color lists.


# 38. Plotting scatter pl
import matplotlib.dates as mdates

# Method
# In method 1, the color for each data point is determined by the date value in date_point. The date values are converted to numerical values using mdates.date2num() before passing to ax.scatter(). The colorbar ticks and labels are set using mdates.AutoDateLocator() and mdates.ConciseDateFormatter(). 1
fig1 = plt.figure(figsize=(5, 5))
ax1 = fig1.add_subplot()

date_point = df['Date']
ss1 = ax1.scatter(df['WSOC'], df['OC'], c=mdates.date2num(date_point), cmap=plt.cm.coolwarm, s=100)
cbaxes1 = fig1.add_axes([0.29, 0.27, 0.48, 0.02]) 
cb1 = plt.colorbar(ss1, cax=cbaxes1, orientation='horizontal')
loc1 = mdates.AutoDateLocator(maxticks=12)
cb1.ax.xaxis.set_major_locator(loc1)
cb1.ax.xaxis.set_major_formatter(mdates.ConciseDateFormatter(loc1))

# Method 2
# Method 2 uses the datetime values directly as color values, which means the colorbar ticks will match the dates exactly. It manually divides the range of dates into 5 segments and creates tick labels with the abbreviated month names.
fig2 = plt.figure(figsize=(5, 5))
ax2 = fig2.add_subplot()

date_point = df['Date']
ss2 = ax2.scatter(df['WSOC'], df['OC'], c=date_point, cmap=plt.cm.coolwarm, s=100)

cbaxes2 = fig2.add_axes([0.29, 0.27, 0.48, 0.02]) 
cbar2 = plt.colorbar(ss2, cax=cbaxes2, orientation='horizontal')

n = 5
st_po = []
for i in range(0, n, 1):
    st_po.append(np.array_split(pd.to_datetime(sorted(date_point)), n)[i].values[0])

cb_ticks = [float(c) for c in st_po]
cbar2.ax.set_xticklabels(pd.to_datetime(cb_ticks).strftime('%b')) #B     
cbar2.ax.tick_params(labelsize=8.5)

#39. plotting color palette based on user-defined color list
from matplotlib.colors import ListedColormap
import matplotlib as mpl

fig = plt.subplots(figsize=(10,2))
ax = plt.subplot(111)
color_list = ['#cccccc', '#da6701','#00994d','#00cccc', '#0000ff','#000099','#c9ace6','#9039e6','#000000']
cmap_diy = ListedColormap(color_list, color_list)
col_map = cmap_diy
new_val = []
ticks = np.linspace(0.0,1, len(color_list)+1)
for i in range(0,len(ticks)-1,1):
    new_val.append((ticks[i]+ticks[i+1])/2.0)
cbar  = mpl.colorbar.ColorbarBase(ax, cmap=col_map, orientation = 'horizontal', ticks =new_val, alpha = 0.75)
cbar.ax.set_xticklabels(color_list, fontsize =12, rotation = 30) 
ttl = plt.title('EESI Color Palette 1',fontweight="bold",fontsize =18,)
ttl.set_position([.5, 1.15])
plt.tight_layout()

#40. Boxplot
def box_plot(ax, data, pos, color):
    bp = ax.boxplot(data, sym='', whis=[5, 95], widths=(len(data) + 4) / (2 * len(data)) * 0.7, positions=pos, 
                    boxprops=dict(facecolor=color, edgecolor=color), medianprops=dict(color='k'), 
                    whiskerprops=dict(color='k'), capprops=dict(color='k'), patch_artist=True)
    for box in bp['boxes']:
        box.set(facecolor=color)
    return bp

#41. Linear fitting in short
# Remove NaN values from the data
import stats
mask = ~np.isnan(x) & ~np.isnan(y)
x_clean = x[mask]
y_clean = y[mask]

# Perform linear regression and calculate the slope and intercept
slope, intercept, r_value, p_value, std_err = stats.linregress(x_clean, y_clean)
# Create a scatter plot with error bars using matplotlib
fig, ax = plt.subplots()
ax.scatter(x_clean, y_clean, s=50, edgecolors='black', alpha=0.8)
# Add the linear fit line
ax.plot(x_clean, slope*x_clean + intercept, color='red', linestyle='-', label='Linear Fit')
# Annotate the fitted equation
ax.text(0.05, 0.95, 'y = {:.2f}x + {:.2f}'.format(slope, intercept), transform=ax.transAxes, fontsize=14,horizontalalignment='left',\ verticalalignment='top', bbox=dict(facecolor='white',edgecolor='white', alpha=0.8))

#42. Color-coded with dates
fig  = plt.figure(figsize=(12,4))
c_time   = mdates.date2num(auto_df_dl['Date'])
ss       = plt.scatter(pd.to_datetime(auto_df_dl['sample_starting_time']), auto_df_dl['130.15903'], c=c_time,cmap = plt.cm.Spectral_r)
cbaxes = fig.add_axes([0.61, 0.75, 0.25,0.02]) 
cb = plt.colorbar(ss, cax=cbaxes, orientation='horizontal')
loc = mdates.AutoDateLocator(maxticks=12)
cb.ax.xaxis.set_major_locator(loc)
cb.ax.xaxis.set_major_formatter(mdates.ConciseDateFormatter(loc))


# 43. Add tooltips to the scatter plots
labels = ["%.2f" % (c) + tof_ion[i] for i, c in enumerate(tof_mz.values)]
tooltip = mpld3.plugins.PointLabelTooltip(ax.collections, labels=labels)
mpld3.plugins.connect(fig, tooltip)


# 44. Horizontial and vertical color bar
cbaxes = fig.add_axes([0.775, 0.2, 0.11, 0.015]) 
ticks = [50,100, 250,500] 
cbar = plt.colorbar(ss_,ticks =ticks, cax=cbaxes,orientation='horizontal', )
cbar.ax.set_xticks(ticks) # set the tick locations
cbar.ax.set_xticklabels(tick_labels, fontsize=7) # set the tick labels
cbar.ax.set_xlabel('FRP (MW)',fontsize = 8, labelpad = -25)
cbar.ax.tick_params(color='#FFFFFF', direction='in')

# 45. Dendragram
def plot_matrix_dendrogram_cn(corr_,va):
    D = corr_
    ax1 = fig.add_axes([0.9,0.15,0.11,0.7*7/6.0])
    Y = sch.linkage(D, method='ward') #method='centroid')ward
    set_link_color_palette(line_color)
    # Temporarily override the default line width:
    with plt.rc_context({'lines.linewidth': 1.25}):
        Z1 = sch.dendrogram(Y, orientation='right',above_threshold_color='grey', color_threshold=va,
                           )
    ax1.set_xticks([])
    ax1.set_yticks([])
    ax1.axis('off')
    
    axmatrix = fig.add_axes([0.05,0.15,0.70,0.7*7/6.0])
    idx1 = Z1['leaves']
    idx2 = Z1['leaves']
    D = D[idx1,:]
    D = D[:,idx1]
    mask =  np.tri(D.shape[1], k=-1)
    A = np.ma.array(D, mask=mask)
    cmap =  plt.cm.YlOrRd
#     cmap = cm.get_cmap('YlGnBu') #YlOrRd
    cmap.set_over('#C0C1C0')
    im = axmatrix.matshow(A, aspect='auto', origin='lower', cmap=cmap,vmax = 0.999, )#Spectral_r
#     im = axmatrix.pcolormesh(A, cmap=cmap,vmax = 0.999, edgecolors ='grey',lw = 0.0)#Spectral_r    
#     axmatrix.set_xticks(range(len(ele_name))-0.5)
    axmatrix.set_xticks(np.arange(0,len(ele_name),1) - 0.5)
    site_X = [ele_name[i] for i in idx1]
    axmatrix.set_xticklabels(site_X, minor=False, va="baseline")
    axmatrix.xaxis.set_label_position('bottom')
    axmatrix.xaxis.tick_bottom()
    pylab.xticks(fontsize=8,)
    axmatrix.tick_params(axis='x', pad=60,length=0,rotation = 45)
    axmatrix.tick_params(axis='y', pad=2,length=0)    

    ind_array = np.arange(0,len(ele_name),1)
    x, y = np.meshgrid(ind_array, ind_array)
    for i in range(0,len(ele_name),1):
        for j in range(0,len(ele_name),1):
            if math.isnan(A[j,i]) == 0:
                axmatrix.text(i, j, str(round(A[j,i], 2)), color = 'black',va='center', ha='center', fontsize = 12)

    axcolor = fig.add_axes([0.05,0.7,0.2,0.015])
    cbar = pylab.colorbar(im, cax=axcolor,orientation='horizontal' )
    cbar.set_label(r'$\mathregular{{Pearson^{\prime}}s\ r}$', labelpad = -35, fontsize = 10)    
    loc_ = np.array([0.2,0.5,0.8])
    cbar.set_ticks(loc_)
    ttt = [0.2,0.5,0.8]
    cbar.set_ticklabels(ttt,)
    cbar.ax.tick_params(color="w", direction='in',labelsize=10)# length

    axmatrix.set_yticks(range(len(ele_name)))
    site_Y = [ ele_name[i] for i in idx1]
    axmatrix.set_yticklabels(site_Y, minor=False, fontsize =8)
    axmatrix.yaxis.set_label_position('right')
    axmatrix.yaxis.tick_right()
    pylab.yticks( fontsize=8)
    axmatrix.spines['left'].set_visible(False)
    axmatrix .spines['top'].set_visible(False)  
    
    
def fitting_line(x,y):
    mask = ~np.isnan(x) & ~np.isnan(y)
    x_clean = x[mask]
    y_clean = y[mask]

    slope, intercept, r_value, p_value, std_err = stats.linregress(x_clean, y_clean)
    annotation = f"y = {slope:.2f}x + {intercept:.2f}\n\n" + f"r = {r_value:.2f}"
    ax.annotate(annotation, xy=(0.05, 0.7), xycoords='axes fraction', fontsize = 10)
    # the version without slope
    # x_clean = x_clean[:,np.newaxis]
    # a, _, _, _ = np.linalg.lstsq(x_clean, y_clean)
    # print (a,)
    # annotation = f"y = {a[0]:.2f}x"
    # ax.annotate(annotation, xy=(0.15, 0.7), xycoords='axes fraction', fontsize = 10)


## 46. Venn plot for 2 elements
list_a,list_b =  owb_poa,owb_soa
ions_intersection      = list_a[list_a['Ion'].isin(list_b['Ion'])]['Ion'].unique()
ions_only_a            = list_a[~list_a['Ion'].isin(list_b['Ion'])]['Ion'].unique()
ions_only_b           =  list_b[~list_b['Ion'].isin(list_a['Ion'])]['Ion'].unique()

from matplotlib_venn import venn2
# Determine the counts
num_ions_in_a_not_b  = len(ions_only_a)
num_ions_in_b_not_a = len(ions_only_b)
num_ions_intersection = len(ions_intersection)

# Plot Venn Diagram
plt.figure(figsize=(6, 5))
venn2(subsets=(num_ions_in_a_not_b,num_ions_in_b_not_a, num_ions_intersection), 
      set_labels=('Open wood burning POA','Open wood burning SOA'),
     set_colors=('#5477b1', '#f5a34f'))
# plt.title("Orbitrap peak list")
plt.savefig('./Figures/20240423/Orbitrap_fitted_ions_OWB_POA+SOA.png', dpi = 400)
plt.tight_layout()
plt.show()


## 47. Venn plot for 3 elements
def only_in_target(tar,ref1,ref2):
    list1= (tar[tar['Ion'].isin(ref1['Ion'])]['Ion'].unique())
    list2= (tar[tar['Ion'].isin(ref2['Ion'])]['Ion'].unique())
    
    list_tot = np.unique(np.append(list1,list2))
    return tar[~tar['Ion'].isin(list_tot)]['Ion'].unique()

# Calculate intersections
ions_intersection_ab = list_a[list_a['Ion'].isin(list_b['Ion'])]['Ion'].unique()
ions_intersection_ac = list_a[list_a['Ion'].isin(list_c['Ion'])]['Ion'].unique()
ions_intersection_bc = list_b[list_b['Ion'].isin(list_c['Ion'])]['Ion'].unique()

# Intersection of all three lists
ions_intersection_abc = list_a[list_a['Ion'].isin(ions_intersection_bc)]['Ion'].unique()


# Calculate unique ions
ions_only_a = only_in_target(list_a,list_b,list_c)
ions_only_b = only_in_target(list_b,list_a,list_c)
ions_only_c = only_in_target(list_c,list_a,list_b)

# Unique in a, b and c but not in the intersection of all three
ions_unique_a = np.setdiff1d(ions_only_a, ions_intersection_abc)
ions_unique_b = np.setdiff1d(ions_only_b, ions_intersection_abc)
ions_unique_c = np.setdiff1d(ions_only_c, ions_intersection_abc)

# Calculate lengths
num_ions_in_a_not_bc = len(ions_unique_a)
num_ions_in_b_not_ac = len(ions_unique_b)
num_ions_in_c_not_ab = len(ions_unique_c)
num_ions_intersection_ab = len(ions_intersection_ab) - len(ions_intersection_abc)
num_ions_intersection_ac = len(ions_intersection_ac) - len(ions_intersection_abc)
num_ions_intersection_bc = len(ions_intersection_bc) - len(ions_intersection_abc)
num_ions_intersection_abc = len(ions_intersection_abc)

from matplotlib_venn import venn3

plt.figure(figsize=(10, 6))
v = venn3(subsets=(num_ions_in_a_not_bc, num_ions_in_b_not_ac, num_ions_intersection_ab,
                   num_ions_in_c_not_ab, num_ions_intersection_ac, num_ions_intersection_bc, num_ions_intersection_abc),
     set_colors=('#a6cee3', '#d83f3f', '#405f3a'),      
      set_labels=('All samples Merged (2371 ions)', '     Coal burning SOA (616 ions)', '                                                    Open burning SOA (858 ions)'))

# change the edgecolor to black
for area in v.patches:
    if area:
        area.set_edgecolor('k')

plt.tight_layout()
plt.savefig("./Figures/20240423/Combined_coal_POA_owb_SOA_All.png", dpi = 400)
plt.show()

4. Jupyter and Python setting

#1. Full width displaying all the time
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))

#2. Import script.py files
import os,sys
scriptpath = "/Users/HYF/Downloads/stackoverflow_tag_cloud-master/"
##Add the directory containing your module to the Python path (wants absolute paths)
sys.path.append(os.path.abspath(scriptpath))
## Do the import
import stackoverflow_users_taginfo
from stackoverflow_users_taginfo import taginfo
info = taginfo(link = 4918632, num_tags = 400)
WordCloud().generate_from_frequencies(info).to_image().save('TagCloud.pdf')

#3. Check the installing path
import time
time.file

#4. Get the pathname of the file
script_dir = os.path.dirname(os.path.realpath(filename)) + os.sep             

#5. Change the size of the uploaded figure
<img src="https://i.stack.imgur.com/kK1LC.png" width="100">

#6. Inquire the information of computer, Python and its package using watermark package
pip install watermark
%load_ext watermark 
## Compiler, system, and CPU
%watermark 
## numpy version
%watermark -p numpy

# 7. 取消Warnings显示
import warnings
warnings.filterwarnings('ignore')

 5. Some tex characters

\leq  <

6. Spatial analysis

# 1. intersection of two lines
import fiona
from shapely.geometry import shape
from shapely.geometry import LineString
line1 = fiona.open(fname)# fname is the path to a specific polyline-format shapefile.
line1 = shape(line1.next()['geometry']) 
start_point, end_point = (111,22),(33,33)
line2 = LineString([start_point, end_point])
print line1.intersection(line2)
print line1.intersection(line2).is_empty

# 2. Read .tif
import warnings
import rasterio
warnings.filterwarnings('ignore')
import skimage.transform as st 
pathToRaster  =r'/Users/HYF/Downloads/2014/per2014/中国年均降水.tif'
xs = np.array([100.5, 12.0])
ys = np.array([2.5, 42.0])
# src.bounds
with rasterio.open(pathToRaster) as src:
    arr = src.read()
    arr[arr<0] = np.nan
    arr = arr[0,:,:]
    rows, cols = rasterio.transform.rowcol(src.transform, xs, ys)    

# 3. mask point series
mask_= []
for i in range(0,len(df_fire_cn),1):
    xy_point = gpd.geoseries.Point(df_fire_cn.lon.iloc[i],df_fire_cn.lat.iloc[i])
    res = geom.contains(xy_point)
    mask_.append(res)
df_fire_cn = df_fire_cn[mask_]    

# 4. Classify scatter point based on polylines
import warnings
warnings.filterwarnings('ignore')
from shapely.geometry import shape
from shapely.geometry import LineString
# loading the boundary layer
import fiona
fname = '/Users/HYF/Dropbox/data/geo/中国地图/供暖分界/N-S_boundary.shp'
line1 = fiona.open(fname)
line1 = shape(line1.next()['geometry']) 
# set a end point which is the southernmost for all stations. 
end_point  = (dy[dy['lat']==dy['lat'].min()]['lon'].values[0],dy[dy['lat']==dy['lat'].min()]['lat'].values[0])
# loop all monitoring stations for classification
dy['NS']= np.nan
for i in range(0,len(dy),1):
    start_point = (dy['lon'].iloc[i],dy['lat'].iloc[i])
    line2       = LineString([start_point, end_point])
    if line1.intersection(line2).is_empty:
        dy["NS"].iloc[i]='S'
    else:
        dy["NS"].iloc[i]='N'        

# 5. transform shp file to geojson
import fiona
import json

# Open the shapefile using fiona
with fiona.open("shapefile.shp") as src:
    # Create a new GeoJSON file
    with open("shapefile.geojson", "w") as output:
        # Write the GeoJSON representation of the shapefile to the file
        output.write(json.dumps(list(src)))

# referencing https://gist.github.com/pelson/9785576
import fiona
import shapely.vectorized
from shapely.geometry import shape
fname = r'xxx/china.shp'
cn_area = fiona.open(fname)
pol = cn_area.next()
geom = shape(pol['geometry'])

## four corner 
x0,x1 = geom.bounds[0],geom.bounds[2]
y0,y1 = geom.bounds[1],geom.bounds[3]

## creating the mask array

x  = np.linspace(x0,x1,30)
y  = np.linspace(y0,y1,20)
xx,yy = np.meshgrid(x,y)
import time
start = time.time()
mask_ = shapely.vectorized.contains(geom,  xx,yy)
print "Process time: " + str(time.time() - start)

## 出现错误:  'Polygon' object is not iterable      
shapeFeature([c.geometry])    

# when nothing showed up in Plotly 
import plotly.io as pio
pio.renderers.default = "notebook"  # or "jupyterlab"

7. Other functions

# 1.Crop PDF file
# https://pypi.org/project/pdfcropper/#files
pdf-crop-margins -v -s -u /Users/HYF/Dropbox/thesis//5_全国时空分析/output/Sec3/As_空间差异图.pdf

# 2. When the self-installed files were unable to be imported
python setup.py build
python setup.py install
## Then, install the package and egg into
>/Users/HYF/anaconda3/lib/python3.7/site-packages

#3. Enabling code folding in Jupyter lab
setting/notebook
{
 "codeCellConfig": {
 "codeFolding": true
 }
}

#4. How to search for content in any notebook on local PC by keywords
import glob
pattern = './../../../../**/*.ipynb'
query = 'EESI_inforamtion_example'

for filepath in glob.iglob(pattern, recursive=True):
    with open(filepath) as file:
        s = file.read()
        if (s.find(query) > -1):
            print(filepath)
            
            
            
# 5. pandas function for reading markdown table
# Paste in your table and assign it to a variable
md_table_string = """
| Risk Factor | Estimated Annual Number of Deaths |
|---|---|
| Air pollution (outdoor & indoor) | 1,670,000 |
| High blood pressure | 1,470,000 |
| High blood sugar | 1,120,000 |
| Smoking | 1,010,000 |
| Outdoor particulate matter pollution | 979,682 |
| Indoor air pollution | 630,093 |
| Obesity | 606,890 |
| Unsafe water source | 508,290 |
| Low birth weight | 579,108 |
| Alcohol use | 343,695 |
| Unsafe sanitation | 258,257 |
| Diet low in whole grains | 256,232 |
| Diet low in fruits | 240,098 |
| Secondhand smoke | 174,797 |
| Diet high in sodium | 173,939 |
| No access to handwashing facility | 164,172 |
| Diet low in nuts and seeds | 134,007 |
| Diet low in vegetables | 118,682 |
| Low bone mineral density | 118,283 |
| Low physical activity | 96,012 |
| Drug use | 72,903 |
| Child stunting | 17,973 |
| Iron deficiency | 10,098 |
| Non-exclusive breastfeeding | 1,501 |
"""

from io import StringIO
death_factor = pd.read_csv(
    StringIO(md_table_string.replace(' ', '')),  # Get rid of whitespaces
    sep='|',
    index_col=1
).dropna(
    axis=1,
    how='all'
).iloc[1:]



## mount the external drive
sudo umount /mnt/n
sudo mount -t drvfs -o metadata,uid=1000,gid=1000 N: /mnt/n


## unzip all files within the folder
#!/bin/bash   \# Navigate to the directory containing the zip files 
cd /path/to/your/directory 
# Loop through all files with .z* extensions and unzip them   
for file in 248650.z*; do
    echo "Unzipping $file..." 
    unzip "$file" -d ./unzip/ \# Unzip to a subdirectory called 'unzipped_files'   
done    
echo "All files have been unzipped."