\documentclass[10pt,a4paper]{article}

% Packages
\usepackage{fancyhdr}           % For header and footer
\usepackage{multicol}           % Allows multicols in tables
\usepackage{tabularx}           % Intelligent column widths
\usepackage{tabulary}           % Used in header and footer
\usepackage{hhline}             % Border under tables
\usepackage{graphicx}           % For images
\usepackage{xcolor}             % For hex colours
%\usepackage[utf8x]{inputenc}    % For unicode character support
\usepackage[T1]{fontenc}        % Without this we get weird character replacements
\usepackage{colortbl}           % For coloured tables
\usepackage{setspace}           % For line height
\usepackage{lastpage}           % Needed for total page number
\usepackage{seqsplit}           % Splits long words.
%\usepackage{opensans}          % Can't make this work so far. Shame. Would be lovely.
\usepackage[normalem]{ulem}     % For underlining links
% Most of the following are not required for the majority
% of cheat sheets but are needed for some symbol support.
\usepackage{amsmath}            % Symbols
\usepackage{MnSymbol}           % Symbols
\usepackage{wasysym}            % Symbols
%\usepackage[english,german,french,spanish,italian]{babel}              % Languages

% Document Info
\author{mitcht}
\pdfinfo{
  /Title (pyspark.pdf)
  /Creator (Cheatography)
  /Author (mitcht)
  /Subject (Pyspark Cheat Sheet)
}

% Lengths and widths
\addtolength{\textwidth}{6cm}
\addtolength{\textheight}{-1cm}
\addtolength{\hoffset}{-3cm}
\addtolength{\voffset}{-2cm}
\setlength{\tabcolsep}{0.2cm} % Space between columns
\setlength{\headsep}{-12pt} % Reduce space between header and content
\setlength{\headheight}{85pt} % If less, LaTeX automatically increases it
\renewcommand{\footrulewidth}{0pt} % Remove footer line
\renewcommand{\headrulewidth}{0pt} % Remove header line
\renewcommand{\seqinsert}{\ifmmode\allowbreak\else\-\fi} % Hyphens in seqsplit
% This two commands together give roughly
% the right line height in the tables
\renewcommand{\arraystretch}{1.3}
\onehalfspacing

% Commands
\newcommand{\SetRowColor}[1]{\noalign{\gdef\RowColorName{#1}}\rowcolor{\RowColorName}} % Shortcut for row colour
\newcommand{\mymulticolumn}[3]{\multicolumn{#1}{>{\columncolor{\RowColorName}}#2}{#3}} % For coloured multi-cols
\newcolumntype{x}[1]{>{\raggedright}p{#1}} % New column types for ragged-right paragraph columns
\newcommand{\tn}{\tabularnewline} % Required as custom column type in use

% Font and Colours
\definecolor{HeadBackground}{HTML}{333333}
\definecolor{FootBackground}{HTML}{666666}
\definecolor{TextColor}{HTML}{333333}
\definecolor{DarkBackground}{HTML}{80060A}
\definecolor{LightBackground}{HTML}{FBF7F7}
\renewcommand{\familydefault}{\sfdefault}
\color{TextColor}

% Header and Footer
\pagestyle{fancy}
\fancyhead{} % Set header to blank
\fancyfoot{} % Set footer to blank
\fancyhead[L]{
\noindent
\begin{multicols}{3}
\begin{tabulary}{5.8cm}{C}
    \SetRowColor{DarkBackground}
    \vspace{-7pt}
    {\parbox{\dimexpr\textwidth-2\fboxsep\relax}{\noindent
        \hspace*{-6pt}\includegraphics[width=5.8cm]{/web/www.cheatography.com/public/images/cheatography_logo.pdf}}
    }
\end{tabulary}
\columnbreak
\begin{tabulary}{11cm}{L}
    \vspace{-2pt}\large{\bf{\textcolor{DarkBackground}{\textrm{Pyspark Cheat Sheet}}}} \\
    \normalsize{by \textcolor{DarkBackground}{mitcht} via \textcolor{DarkBackground}{\uline{cheatography.com/50563/cs/14121/}}}
\end{tabulary}
\end{multicols}}

\fancyfoot[L]{ \footnotesize
\noindent
\begin{multicols}{3}
\begin{tabulary}{5.8cm}{LL}
  \SetRowColor{FootBackground}
  \mymulticolumn{2}{p{5.377cm}}{\bf\textcolor{white}{Cheatographer}}  \\
  \vspace{-2pt}mitcht \\
  \uline{cheatography.com/mitcht} \\
  \end{tabulary}
\vfill
\columnbreak
\begin{tabulary}{5.8cm}{L}
  \SetRowColor{FootBackground}
  \mymulticolumn{1}{p{5.377cm}}{\bf\textcolor{white}{Cheat Sheet}}  \\
   \vspace{-2pt}Not Yet Published.\\
   Updated 29th December, 2017.\\
   Page {\thepage} of \pageref{LastPage}.
\end{tabulary}
\vfill
\columnbreak
\begin{tabulary}{5.8cm}{L}
  \SetRowColor{FootBackground}
  \mymulticolumn{1}{p{5.377cm}}{\bf\textcolor{white}{Sponsor}}  \\
  \SetRowColor{white}
  \vspace{-5pt}
  %\includegraphics[width=48px,height=48px]{dave.jpeg}
  Measure your website readability!\\
  www.readability-score.com
\end{tabulary}
\end{multicols}}




\begin{document}
\raggedright
\raggedcolumns

% Set font size to small. Switch to any value
% from this page to resize cheat sheet text:
% www.emerson.emory.edu/services/latex/latex_169.html
\footnotesize % Small font.

\begin{multicols*}{2}

\begin{tabularx}{8.4cm}{X}
\SetRowColor{DarkBackground}
\mymulticolumn{1}{x{8.4cm}}{\bf\textcolor{white}{Read / Write .csv}}  \tn
\SetRowColor{LightBackground}
\mymulticolumn{1}{x{8.4cm}}{df = \seqsplit{(sqlContext.read.format("com.databricks.spark}.csv")\textbackslash{} \newline        .option("header", "true")\textbackslash{} \newline        .option("inferschema", "true")\textbackslash{} \newline        .option("mode", "DROPMALFORMED")\textbackslash{} \newline        \seqsplit{.load("hdfs://file.csv"))} \newline  \newline df.write.mode('overwrite'). \newline          option("header", "true"). \newline          \seqsplit{csv("file://filename.csv")}} \tn 
\hhline{>{\arrayrulecolor{DarkBackground}}-}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{8.4cm}{x{4.8 cm} x{3.2 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{8.4cm}}{\bf\textcolor{white}{Meta Data}}  \tn
% Row 0
\SetRowColor{LightBackground}
`df.printSchema()` & `df.count()` \tn 
% Row Count 1 (+ 1)
% Row 1
\SetRowColor{white}
`len(df.columns)` & `df.columns` \tn 
% Row Count 2 (+ 1)
% Row 2
\SetRowColor{LightBackground}
\mymulticolumn{2}{x{8.4cm}}{`df.dtypes`} \tn 
% Row Count 3 (+ 1)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{8.4cm}{X}
\SetRowColor{DarkBackground}
\mymulticolumn{1}{x{8.4cm}}{\bf\textcolor{white}{Arrange}}  \tn
\SetRowColor{LightBackground}
\mymulticolumn{1}{x{8.4cm}}{\seqsplit{df.withColumnRenamed("col"},"newcol") \newline df.orderBy({[}'var1', 'var2'{]}, ascending = {[}True, False{]}) \newline df.orderBy(df.var1, df.var2.desc())} \tn 
\hhline{>{\arrayrulecolor{DarkBackground}}-}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{8.4cm}{X}
\SetRowColor{DarkBackground}
\mymulticolumn{1}{x{8.4cm}}{\bf\textcolor{white}{Filter}}  \tn
\SetRowColor{LightBackground}
\mymulticolumn{1}{x{8.4cm}}{df.filter(df.var \textgreater{} 10000) \newline df.select('col1', 'col2') \newline df{[}collist{]} \# collist = {[}'var1', ...{]} \newline df.head(5) / df.show(5, truncate=True) / df.take(5) \newline df.drop('var') \newline df.distinct() \newline df.dropDuplicates() \newline df.dropna(subset='var') / df.na.drop() \newline df.isNotNull() \newline df.var.isin("level1", "level2") \newline df.var.like("string") \newline df.var.startswith("m") / df.var.endswith("m") \newline df.sample(False {\emph{with replacement}}, 0.5 {\emph{fraction}}, 12345 {\emph{seed}})} \tn 
\hhline{>{\arrayrulecolor{DarkBackground}}-}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{8.4cm}{x{4 cm} x{4 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{8.4cm}}{\bf\textcolor{white}{Useful Functions}}  \tn
% Row 0
\SetRowColor{LightBackground}
\seqsplit{`.describe('optional\_var')`} & `.count()` \tn 
% Row Count 2 (+ 2)
% Row 1
\SetRowColor{white}
`.show()` & `.fillna(value)` \tn 
% Row Count 3 (+ 1)
% Row 2
\SetRowColor{LightBackground}
`.min() / .max()` & `.mean()` \tn 
% Row Count 4 (+ 1)
% Row 3
\SetRowColor{white}
`.stdev() / .variance()` & `.substr(1,3)` \tn 
% Row Count 6 (+ 2)
% Row 4
\SetRowColor{LightBackground}
\mymulticolumn{2}{x{8.4cm}}{`F.when(df.var \textgreater{} 30, "Y").otherwise("N")`} \tn 
% Row Count 7 (+ 1)
% Row 5
\SetRowColor{white}
\mymulticolumn{2}{x{8.4cm}}{`df.var.alias('newvar') \# used to rename something`} \tn 
% Row Count 9 (+ 2)
% Row 6
\SetRowColor{LightBackground}
`df.cache() / df.unpersist()` & `.cast('Double')` \tn 
% Row Count 11 (+ 2)
% Row 7
\SetRowColor{white}
`.replace(10, 20) \# ????` & `.na.fill(0, subset = 'var')` \tn 
% Row Count 13 (+ 2)
% Row 8
\SetRowColor{LightBackground}
\mymulticolumn{2}{x{8.4cm}}{`time()`} \tn 
% Row Count 14 (+ 1)
% Row 9
\SetRowColor{white}
\mymulticolumn{2}{x{8.4cm}}{`from pyspark.sql import functions as F`} \tn 
% Row Count 15 (+ 1)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{8.4cm}{X}
\SetRowColor{DarkBackground}
\mymulticolumn{1}{x{8.4cm}}{\bf\textcolor{white}{Write Functions / UDF}}  \tn
\SetRowColor{LightBackground}
\mymulticolumn{1}{x{8.4cm}}{from pyspark.sql.functions import udf \newline F1 = udf(lambda x: '-1' if condition else x, StringType()) \# NB return type \newline df = df.withColumn('newvar', F1(df{[}'invar'{]}))} \tn 
\hhline{>{\arrayrulecolor{DarkBackground}}-}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{8.4cm}{X}
\SetRowColor{DarkBackground}
\mymulticolumn{1}{x{8.4cm}}{\bf\textcolor{white}{Applying Functions}}  \tn
\SetRowColor{LightBackground}
\mymulticolumn{1}{x{8.4cm}}{\seqsplit{df.select('val').map(lambda} x: x*2)} \tn 
\hhline{>{\arrayrulecolor{DarkBackground}}-}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{8.4cm}{X}
\SetRowColor{DarkBackground}
\mymulticolumn{1}{x{8.4cm}}{\bf\textcolor{white}{Summarise}}  \tn
\SetRowColor{LightBackground}
\mymulticolumn{1}{x{8.4cm}}{df.crosstab('col1', 'col2') \# pair-wise count \newline df.groupby('var').function() \newline df.groupby('var').agg(\{'val' : 'mean'\})} \tn 
\hhline{>{\arrayrulecolor{DarkBackground}}-}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{8.4cm}{X}
\SetRowColor{DarkBackground}
\mymulticolumn{1}{x{8.4cm}}{\bf\textcolor{white}{Join}}  \tn
\SetRowColor{LightBackground}
\mymulticolumn{1}{x{8.4cm}}{df3 = df1.join(df2, {[}df1.var1 == df2.var1, df1.var2 == df2.var2{]}, 'left') \newline xtra = \seqsplit{df1.select('var').subtract(df2.select('var'))} \# anti-join} \tn 
\hhline{>{\arrayrulecolor{DarkBackground}}-}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{8.4cm}{X}
\SetRowColor{DarkBackground}
\mymulticolumn{1}{x{8.4cm}}{\bf\textcolor{white}{Method Chaining}}  \tn
\SetRowColor{LightBackground}
\mymulticolumn{1}{x{8.4cm}}{df \newline       .select("col1","col2","col3", ...) \newline       .filter(df.col1 \textgreater{} 30 ) \newline       .show()} \tn 
\hhline{>{\arrayrulecolor{DarkBackground}}-}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{8.4cm}{X}
\SetRowColor{DarkBackground}
\mymulticolumn{1}{x{8.4cm}}{\bf\textcolor{white}{New Variable / Column}}  \tn
\SetRowColor{LightBackground}
\mymulticolumn{1}{x{8.4cm}}{df.withColumn('varnew', df.var / 2.0)} \tn 
\hhline{>{\arrayrulecolor{DarkBackground}}-}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{8.4cm}{X}
\SetRowColor{DarkBackground}
\mymulticolumn{1}{x{8.4cm}}{\bf\textcolor{white}{To SQL / Pandas}}  \tn
\SetRowColor{LightBackground}
\mymulticolumn{1}{x{8.4cm}}{\seqsplit{df.registerAsTable('df\_tbl')} \newline sqlContext.sql('select var from df\_tbl').show(5) \newline df.toPandas()} \tn 
\hhline{>{\arrayrulecolor{DarkBackground}}-}
\end{tabularx}
\par\addvspace{1.3em}


% That's all folks
\end{multicols*}

\end{document}