\documentclass[10pt,a4paper]{article} % Packages \usepackage{fancyhdr} % For header and footer \usepackage{multicol} % Allows multicols in tables \usepackage{tabularx} % Intelligent column widths \usepackage{tabulary} % Used in header and footer \usepackage{hhline} % Border under tables \usepackage{graphicx} % For images \usepackage{xcolor} % For hex colours %\usepackage[utf8x]{inputenc} % For unicode character support \usepackage[T1]{fontenc} % Without this we get weird character replacements \usepackage{colortbl} % For coloured tables \usepackage{setspace} % For line height \usepackage{lastpage} % Needed for total page number \usepackage{seqsplit} % Splits long words. %\usepackage{opensans} % Can't make this work so far. Shame. Would be lovely. \usepackage[normalem]{ulem} % For underlining links % Most of the following are not required for the majority % of cheat sheets but are needed for some symbol support. \usepackage{amsmath} % Symbols \usepackage{MnSymbol} % Symbols \usepackage{wasysym} % Symbols %\usepackage[english,german,french,spanish,italian]{babel} % Languages % Document Info \author{woobidoobi} \pdfinfo{ /Title (dl.pdf) /Creator (Cheatography) /Author (woobidoobi) /Subject (DL Cheat Sheet) } % Lengths and widths \addtolength{\textwidth}{6cm} \addtolength{\textheight}{-1cm} \addtolength{\hoffset}{-3cm} \addtolength{\voffset}{-2cm} \setlength{\tabcolsep}{0.2cm} % Space between columns \setlength{\headsep}{-12pt} % Reduce space between header and content \setlength{\headheight}{85pt} % If less, LaTeX automatically increases it \renewcommand{\footrulewidth}{0pt} % Remove footer line \renewcommand{\headrulewidth}{0pt} % Remove header line \renewcommand{\seqinsert}{\ifmmode\allowbreak\else\-\fi} % Hyphens in seqsplit % This two commands together give roughly % the right line height in the tables \renewcommand{\arraystretch}{1.3} \onehalfspacing % Commands \newcommand{\SetRowColor}[1]{\noalign{\gdef\RowColorName{#1}}\rowcolor{\RowColorName}} % Shortcut for row colour \newcommand{\mymulticolumn}[3]{\multicolumn{#1}{>{\columncolor{\RowColorName}}#2}{#3}} % For coloured multi-cols \newcolumntype{x}[1]{>{\raggedright}p{#1}} % New column types for ragged-right paragraph columns \newcommand{\tn}{\tabularnewline} % Required as custom column type in use % Font and Colours \definecolor{HeadBackground}{HTML}{333333} \definecolor{FootBackground}{HTML}{666666} \definecolor{TextColor}{HTML}{333333} \definecolor{DarkBackground}{HTML}{8048C9} \definecolor{LightBackground}{HTML}{F7F3FB} \renewcommand{\familydefault}{\sfdefault} \color{TextColor} % Header and Footer \pagestyle{fancy} \fancyhead{} % Set header to blank \fancyfoot{} % Set footer to blank \fancyhead[L]{ \noindent \begin{multicols}{3} \begin{tabulary}{5.8cm}{C} \SetRowColor{DarkBackground} \vspace{-7pt} {\parbox{\dimexpr\textwidth-2\fboxsep\relax}{\noindent \hspace*{-6pt}\includegraphics[width=5.8cm]{/web/www.cheatography.com/public/images/cheatography_logo.pdf}} } \end{tabulary} \columnbreak \begin{tabulary}{11cm}{L} \vspace{-2pt}\large{\bf{\textcolor{DarkBackground}{\textrm{DL Cheat Sheet}}}} \\ \normalsize{by \textcolor{DarkBackground}{woobidoobi} via \textcolor{DarkBackground}{\uline{cheatography.com/99373/cs/21014/}}} \end{tabulary} \end{multicols}} \fancyfoot[L]{ \footnotesize \noindent \begin{multicols}{3} \begin{tabulary}{5.8cm}{LL} \SetRowColor{FootBackground} \mymulticolumn{2}{p{5.377cm}}{\bf\textcolor{white}{Cheatographer}} \\ \vspace{-2pt}woobidoobi \\ \uline{cheatography.com/woobidoobi} \\ \end{tabulary} \vfill \columnbreak \begin{tabulary}{5.8cm}{L} \SetRowColor{FootBackground} \mymulticolumn{1}{p{5.377cm}}{\bf\textcolor{white}{Cheat Sheet}} \\ \vspace{-2pt}Not Yet Published.\\ Updated 5th November, 2019.\\ Page {\thepage} of \pageref{LastPage}. \end{tabulary} \vfill \columnbreak \begin{tabulary}{5.8cm}{L} \SetRowColor{FootBackground} \mymulticolumn{1}{p{5.377cm}}{\bf\textcolor{white}{Sponsor}} \\ \SetRowColor{white} \vspace{-5pt} %\includegraphics[width=48px,height=48px]{dave.jpeg} Measure your website readability!\\ www.readability-score.com \end{tabulary} \end{multicols}} \begin{document} \raggedright \raggedcolumns % Set font size to small. Switch to any value % from this page to resize cheat sheet text: % www.emerson.emory.edu/services/latex/latex_169.html \footnotesize % Small font. \begin{multicols*}{3} \begin{tabularx}{5.377cm}{x{1.14425 cm} x{0.96117 cm} x{2.47158 cm} } \SetRowColor{DarkBackground} \mymulticolumn{3}{x{5.377cm}}{\bf\textcolor{white}{Session}} \tn % Row 0 \SetRowColor{LightBackground} \seqsplit{Interpreter} & \seqsplit{\%jdbc(hive)} & \%livy2.pyspark \tn % Row Count 2 (+ 2) % Row 1 \SetRowColor{white} Create Session & & "from pyspark\_llap import HiveWarehouseSession hive = \seqsplit{HiveWarehouseSession.session(spark).build()"} \tn % Row Count 7 (+ 5) % Row 2 \SetRowColor{LightBackground} Connect to database & use \seqsplit{ampb\_sandbox} & \seqsplit{"hive.setDatabase(""dl\_prod\_sandbox\_agi"")} " \tn % Row Count 10 (+ 3) % Row 3 \SetRowColor{white} List Databases & & hive.execute("show \seqsplit{databases").show(truncate=False)} \tn % Row Count 13 (+ 3) % Row 4 \SetRowColor{LightBackground} List Tables & show tables & hive.execute("show \seqsplit{tables").show(truncate=False)} \tn % Row Count 16 (+ 3) \hhline{>{\arrayrulecolor{DarkBackground}}---} \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{5.377cm}{x{0.82386 cm} x{1.87657 cm} x{1.87657 cm} } \SetRowColor{DarkBackground} \mymulticolumn{3}{x{5.377cm}}{\bf\textcolor{white}{Data wrangling}} \tn % Row 0 \SetRowColor{LightBackground} {\bf{function}} & {\bf{ sql equivalient }} & {\bf{ pyspark df }} \tn % Row Count 2 (+ 2) % Row 1 \SetRowColor{white} \seqsplit{Selecting} \seqsplit{columns} & select \seqsplit{arbetsgivaravgift\_organisationsnummer\_id} as orgnr, \seqsplit{arbetsgivaravgift\_redovisad\_period} as period, \seqsplit{arbetsgivaravgift\_inbetalat\_belopp} as \seqsplit{inbetalt\_belopp} from df\_agavgift & \seqsplit{df\_agavgift.select(} \seqsplit{col("arbetsgivaravgift\_organisationsnummer\_id")}.alias("orgnr"), \seqsplit{col("arbetsgivaravgift\_redovisad\_period")}.alias("period"), \seqsplit{col("arbetsgivaravgift\_inbetalat\_belopp")}.alias("inbetalat\_belopp") ) \tn % Row Count 18 (+ 16) \hhline{>{\arrayrulecolor{DarkBackground}}---} \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{5.377cm}{x{2.4885 cm} x{2.4885 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{5.377cm}}{\bf\textcolor{white}{Livy2.pyspark}} \tn % Row 0 \SetRowColor{LightBackground} Transfer data from hive to pyspark dataframe & pyspark\_df = \seqsplit{hive.table("hive\_table")} \tn % Row Count 3 (+ 3) % Row 1 \SetRowColor{white} Transfer data from pyspark to hive & \seqsplit{pyspark\_df.registerTempTable("hive\_table")} \tn % Row Count 6 (+ 3) % Row 2 \SetRowColor{LightBackground} Remove a temp table & hive\_table.drop() \tn % Row Count 7 (+ 1) \hhline{>{\arrayrulecolor{DarkBackground}}--} \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{5.377cm}{x{1.9908 cm} x{2.9862 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{5.377cm}}{\bf\textcolor{white}{Combining data}} \tn % Row 0 \SetRowColor{LightBackground} inner join & df1.join(df2, df1.name == df2.name) \tn % Row Count 2 (+ 2) % Row 1 \SetRowColor{white} left join & df1.join(df2, df1.name == df2.name,how='left') \tn % Row Count 4 (+ 2) % Row 2 \SetRowColor{LightBackground} right join & ta.join(tb, ta.name == tb.name,how='right') \tn % Row Count 6 (+ 2) % Row 3 \SetRowColor{white} joining on multiple columns & df1.join(df2, df1.name == df2.name,how='right') \tn % Row Count 8 (+ 2) \hhline{>{\arrayrulecolor{DarkBackground}}--} \SetRowColor{LightBackground} \mymulticolumn{2}{x{5.377cm}}{in pyspark you need to start by setting an alias for the tables that you want to join \newline df1 = TableA.alias('df1') \newline df2 = TableB.alias('df2') \newline \newline \newline Useful links: \newline http://www.learnbymarketing.com/1100/pyspark-joins-by-example/ \newline http://www.learnbymarketing.com/618/pyspark-rdd-basics-examples/} \tn \hhline{>{\arrayrulecolor{DarkBackground}}--} \end{tabularx} \par\addvspace{1.3em} % That's all folks \end{multicols*} \end{document}