\documentclass[10pt,a4paper]{article}

% Packages
\usepackage{fancyhdr}           % For header and footer
\usepackage{multicol}           % Allows multicols in tables
\usepackage{tabularx}           % Intelligent column widths
\usepackage{tabulary}           % Used in header and footer
\usepackage{hhline}             % Border under tables
\usepackage{graphicx}           % For images
\usepackage{xcolor}             % For hex colours
%\usepackage[utf8x]{inputenc}    % For unicode character support
\usepackage[T1]{fontenc}        % Without this we get weird character replacements
\usepackage{colortbl}           % For coloured tables
\usepackage{setspace}           % For line height
\usepackage{lastpage}           % Needed for total page number
\usepackage{seqsplit}           % Splits long words.
%\usepackage{opensans}          % Can't make this work so far. Shame. Would be lovely.
\usepackage[normalem]{ulem}     % For underlining links
% Most of the following are not required for the majority
% of cheat sheets but are needed for some symbol support.
\usepackage{amsmath}            % Symbols
\usepackage{MnSymbol}           % Symbols
\usepackage{wasysym}            % Symbols
%\usepackage[english,german,french,spanish,italian]{babel}              % Languages

% Document Info
\author{datamansam}
\pdfinfo{
  /Title (comparing-core-pyspark-and-pandas-code.pdf)
  /Creator (Cheatography)
  /Author (datamansam)
  /Subject (Comparing Core Pyspark and Pandas Code Cheat Sheet)
}

% Lengths and widths
\addtolength{\textwidth}{6cm}
\addtolength{\textheight}{-1cm}
\addtolength{\hoffset}{-3cm}
\addtolength{\voffset}{-2cm}
\setlength{\tabcolsep}{0.2cm} % Space between columns
\setlength{\headsep}{-12pt} % Reduce space between header and content
\setlength{\headheight}{85pt} % If less, LaTeX automatically increases it
\renewcommand{\footrulewidth}{0pt} % Remove footer line
\renewcommand{\headrulewidth}{0pt} % Remove header line
\renewcommand{\seqinsert}{\ifmmode\allowbreak\else\-\fi} % Hyphens in seqsplit
% This two commands together give roughly
% the right line height in the tables
\renewcommand{\arraystretch}{1.3}
\onehalfspacing

% Commands
\newcommand{\SetRowColor}[1]{\noalign{\gdef\RowColorName{#1}}\rowcolor{\RowColorName}} % Shortcut for row colour
\newcommand{\mymulticolumn}[3]{\multicolumn{#1}{>{\columncolor{\RowColorName}}#2}{#3}} % For coloured multi-cols
\newcolumntype{x}[1]{>{\raggedright}p{#1}} % New column types for ragged-right paragraph columns
\newcommand{\tn}{\tabularnewline} % Required as custom column type in use

% Font and Colours
\definecolor{HeadBackground}{HTML}{333333}
\definecolor{FootBackground}{HTML}{666666}
\definecolor{TextColor}{HTML}{333333}
\definecolor{DarkBackground}{HTML}{A3A3A3}
\definecolor{LightBackground}{HTML}{F3F3F3}
\renewcommand{\familydefault}{\sfdefault}
\color{TextColor}

% Header and Footer
\pagestyle{fancy}
\fancyhead{} % Set header to blank
\fancyfoot{} % Set footer to blank
\fancyhead[L]{
\noindent
\begin{multicols}{3}
\begin{tabulary}{5.8cm}{C}
    \SetRowColor{DarkBackground}
    \vspace{-7pt}
    {\parbox{\dimexpr\textwidth-2\fboxsep\relax}{\noindent
        \hspace*{-6pt}\includegraphics[width=5.8cm]{/web/www.cheatography.com/public/images/cheatography_logo.pdf}}
    }
\end{tabulary}
\columnbreak
\begin{tabulary}{11cm}{L}
    \vspace{-2pt}\large{\bf{\textcolor{DarkBackground}{\textrm{Comparing Core Pyspark and Pandas Code Cheat Sheet}}}} \\
    \normalsize{by \textcolor{DarkBackground}{datamansam} via \textcolor{DarkBackground}{\uline{cheatography.com/139410/cs/31975/}}}
\end{tabulary}
\end{multicols}}

\fancyfoot[L]{ \footnotesize
\noindent
\begin{multicols}{3}
\begin{tabulary}{5.8cm}{LL}
  \SetRowColor{FootBackground}
  \mymulticolumn{2}{p{5.377cm}}{\bf\textcolor{white}{Cheatographer}}  \\
  \vspace{-2pt}datamansam \\
  \uline{cheatography.com/datamansam} \\
  \end{tabulary}
\vfill
\columnbreak
\begin{tabulary}{5.8cm}{L}
  \SetRowColor{FootBackground}
  \mymulticolumn{1}{p{5.377cm}}{\bf\textcolor{white}{Cheat Sheet}}  \\
   \vspace{-2pt}Published 3rd May, 2022.\\
   Updated 28th May, 2022.\\
   Page {\thepage} of \pageref{LastPage}.
\end{tabulary}
\vfill
\columnbreak
\begin{tabulary}{5.8cm}{L}
  \SetRowColor{FootBackground}
  \mymulticolumn{1}{p{5.377cm}}{\bf\textcolor{white}{Sponsor}}  \\
  \SetRowColor{white}
  \vspace{-5pt}
  %\includegraphics[width=48px,height=48px]{dave.jpeg}
  Measure your website readability!\\
  www.readability-score.com
\end{tabulary}
\end{multicols}}




\begin{document}
\raggedright
\raggedcolumns

% Set font size to small. Switch to any value
% from this page to resize cheat sheet text:
% www.emerson.emory.edu/services/latex/latex_169.html
\footnotesize % Small font.

\begin{multicols*}{4}

\begin{tabularx}{3.833cm}{x{1.03122 cm} x{1.00089 cm} x{1.00089 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{3}{x{3.833cm}}{\bf\textcolor{white}{Importing Dataset}}  \tn
% Row 0
\SetRowColor{LightBackground}
\#SPARK titanic\_sp = \seqsplit{spark.table("titanic\_train")} & \#PANDAS titanic\_pd = \seqsplit{titanic\_sp.select("*").toPandas()} & \seqsplit{\#FromPdBackToSPARK} pysparkDF2 = \seqsplit{spark.createDataFrame(pandasDF)} \tn 
% Row Count 5 (+ 5)
\hhline{>{\arrayrulecolor{DarkBackground}}---}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{1.61351 cm} x{1.81949 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{View data in DataFrame}}  \tn
% Row 0
\SetRowColor{LightBackground}
titanic\_sp.show() & display(titanic\_pd) \tn 
% Row Count 1 (+ 1)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{2.02547 cm} x{1.40753 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{Display DataFrame schema}}  \tn
% Row 0
\SetRowColor{LightBackground}
\seqsplit{titanic\_sp.printSchema()} & \seqsplit{titanic\_pd.info()} \tn 
% Row Count 2 (+ 2)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\SetRowColor{LightBackground}
\mymulticolumn{2}{x{3.833cm}}{The column names, column data type, non-null values and Pandas memory use}  \tn 
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{1.7165 cm} x{1.7165 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{Renaming a column in a DataFrame}}  \tn
% Row 0
\SetRowColor{LightBackground}
\seqsplit{column\_renamed=titanic\_sp}.withColumnRenamed("Name","PassengerName").columns & titanic\_pd.rename(columns=\{'Name': 'PassengerName'\}).columns \tn 
% Row Count 4 (+ 4)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{2.12846 cm} x{1.30454 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{View number of columns and rows}}  \tn
% Row 0
\SetRowColor{LightBackground}
\#SPARK \seqsplit{print((titanic\_sp.count()}, \seqsplit{len(titanic\_sp.columns)))} & \#PANDAS \seqsplit{titanic\_pd.shape} \tn 
% Row Count 3 (+ 3)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{1.7165 cm} x{1.7165 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{Dropping Columns}}  \tn
% Row 0
\SetRowColor{LightBackground}
flight\_data = \seqsplit{flight\_data.drop(columns\_to\_drop}, axis = 1) & flight\_data = \seqsplit{flight\_data.drop(*columns\_to\_drop)} \tn 
% Row Count 3 (+ 3)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{1.92248 cm} x{1.51052 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{Unique values of a column}}  \tn
% Row 0
\SetRowColor{LightBackground}
\seqsplit{titanic\_sp.select('Survived').distinct().show()} & titanic\_pd{[}'Survived'{]}.unique() \tn 
% Row Count 3 (+ 3)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{1.7165 cm} x{1.7165 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{View column names}}  \tn
% Row 0
\SetRowColor{LightBackground}
titanic\_sp.columns & titanic\_pd.columns \tn 
% Row Count 1 (+ 1)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{1.7165 cm} x{1.7165 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{Display column datatypes}}  \tn
% Row 0
\SetRowColor{LightBackground}
titanic\_sp.dtypes & titanic\_pd.dtypes \tn 
% Row Count 1 (+ 1)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{1.7165 cm} x{1.7165 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{Convert tyoes}}  \tn
% Row 0
\SetRowColor{LightBackground}
flight\_data = \seqsplit{flight\_data.withColumn('dt\_departure'}, \seqsplit{f.to\_timestamp(flight\_data.departure\_datetime}, 'yyyy-MM-dd HHmm')) & flight\_data{[}'dt\_departure'{]} = pd.to\_datetime(flight\_data{[}'dt\_departure\_datetime'{]}, format='\%Y-\%m-\%d \%H\%M') \tn 
% Row Count 6 (+ 6)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{1.33887 cm} x{2.09413 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{Summary Stats}}  \tn
% Row 0
\SetRowColor{LightBackground}
df.describe() & df.describe().show() \tn 
% Row Count 1 (+ 1)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{1.7165 cm} x{1.7165 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{Aggregation}}  \tn
% Row 0
\SetRowColor{LightBackground}
\seqsplit{df.groupBy("Company")}.agg(\{'Sales':'sum'\}).show() & \seqsplit{pysparkDF.groupBy("gender")} \textbackslash{}          .agg(mean("age"),mean("salary"),max("salary")) \textbackslash{}          .show() \tn 
% Row Count 6 (+ 6)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{1.7165 cm} x{1.7165 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{Filter comparisons}}  \tn
% Row 0
\SetRowColor{LightBackground}
df{[}df{[}'species'{]}.isin({[}'Chinstrap', 'Gentoo'{]}){]}.show(5) & df{[}df{[}'species'{]}.isin({[}'Chinstrap', 'Gentoo'{]}){]}.head() \tn 
% Row Count 3 (+ 3)
% Row 1
\SetRowColor{white}
df{[}df{[}'species'{]}.rlike('G.'){]}.show(5) & df{[}df{[}'species'{]}.str.match('G.'){]}.head() \tn 
% Row Count 5 (+ 2)
% Row 2
\SetRowColor{LightBackground}
df{[}df{[}'flipper'{]}.between(225,229){]}.show(5) & df{[}df{[}'flipper'{]}.between(225,229){]}.head() \tn 
% Row Count 8 (+ 3)
% Row 3
\SetRowColor{white}
df{[}df{[}'mass'{]}.isNull(){]}.show(5) & df{[}df{[}'mass'{]}.isnull(){]}.head() \tn 
% Row Count 10 (+ 2)
% Row 4
\SetRowColor{LightBackground}
df{[}(df{[}'mass'{]}\textless{}3400) \& (df{[}'sex'{]}=='Male'){]}.head() & df{[}(df{[}'mass'{]}\textless{}3400) \& (df{[}'sex'{]}=='Male'){]}.show(5) \tn 
% Row Count 13 (+ 3)
% Row 5
\SetRowColor{white}
df{[}\textasciitilde{}df{[}'flipper'{]}.between(225,229){]}.show(5) & df{[}\textasciitilde{}df{[}'flipper'{]}.between(225,229){]}.head() \tn 
% Row Count 16 (+ 3)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}

\begin{tabularx}{3.833cm}{x{1.7165 cm} x{1.7165 cm} }
\SetRowColor{DarkBackground}
\mymulticolumn{2}{x{3.833cm}}{\bf\textcolor{white}{Conditional Transformations}}  \tn
% Row 0
\SetRowColor{LightBackground}
flight\_data{[}'departure\_delay\_status'{]} = np.where(flight\_data{[}'departure\_delay'{]} \textgreater{} 90, 'Heavy', 'Moderate') & flight\_data = \seqsplit{flight\_data.withColumn('departure\_delay\_status'}, \seqsplit{f.when(flight\_data.departure\_delay} \textgreater{} 90, \seqsplit{'Heavy').otherwise('Moderate')} \tn 
% Row Count 7 (+ 7)
\hhline{>{\arrayrulecolor{DarkBackground}}--}
\end{tabularx}
\par\addvspace{1.3em}


% That's all folks
\end{multicols*}

\end{document}