\documentclass[10pt,a4paper]{article} % Packages \usepackage{fancyhdr} % For header and footer \usepackage{multicol} % Allows multicols in tables \usepackage{tabularx} % Intelligent column widths \usepackage{tabulary} % Used in header and footer \usepackage{hhline} % Border under tables \usepackage{graphicx} % For images \usepackage{xcolor} % For hex colours %\usepackage[utf8x]{inputenc} % For unicode character support \usepackage[T1]{fontenc} % Without this we get weird character replacements \usepackage{colortbl} % For coloured tables \usepackage{setspace} % For line height \usepackage{lastpage} % Needed for total page number \usepackage{seqsplit} % Splits long words. %\usepackage{opensans} % Can't make this work so far. Shame. Would be lovely. \usepackage[normalem]{ulem} % For underlining links % Most of the following are not required for the majority % of cheat sheets but are needed for some symbol support. \usepackage{amsmath} % Symbols \usepackage{MnSymbol} % Symbols \usepackage{wasysym} % Symbols %\usepackage[english,german,french,spanish,italian]{babel} % Languages % Document Info \author{mliafol} \pdfinfo{ /Title (flink.pdf) /Creator (Cheatography) /Author (mliafol) /Subject (Flink Cheat Sheet) } % Lengths and widths \addtolength{\textwidth}{6cm} \addtolength{\textheight}{-1cm} \addtolength{\hoffset}{-3cm} \addtolength{\voffset}{-2cm} \setlength{\tabcolsep}{0.2cm} % Space between columns \setlength{\headsep}{-12pt} % Reduce space between header and content \setlength{\headheight}{85pt} % If less, LaTeX automatically increases it \renewcommand{\footrulewidth}{0pt} % Remove footer line \renewcommand{\headrulewidth}{0pt} % Remove header line \renewcommand{\seqinsert}{\ifmmode\allowbreak\else\-\fi} % Hyphens in seqsplit % This two commands together give roughly % the right line height in the tables \renewcommand{\arraystretch}{1.3} \onehalfspacing % Commands \newcommand{\SetRowColor}[1]{\noalign{\gdef\RowColorName{#1}}\rowcolor{\RowColorName}} % Shortcut for row colour \newcommand{\mymulticolumn}[3]{\multicolumn{#1}{>{\columncolor{\RowColorName}}#2}{#3}} % For coloured multi-cols \newcolumntype{x}[1]{>{\raggedright}p{#1}} % New column types for ragged-right paragraph columns \newcommand{\tn}{\tabularnewline} % Required as custom column type in use % Font and Colours \definecolor{HeadBackground}{HTML}{333333} \definecolor{FootBackground}{HTML}{666666} \definecolor{TextColor}{HTML}{333333} \definecolor{DarkBackground}{HTML}{5FA37A} \definecolor{LightBackground}{HTML}{F5F9F6} \renewcommand{\familydefault}{\sfdefault} \color{TextColor} % Header and Footer \pagestyle{fancy} \fancyhead{} % Set header to blank \fancyfoot{} % Set footer to blank \fancyhead[L]{ \noindent \begin{multicols}{3} \begin{tabulary}{5.8cm}{C} \SetRowColor{DarkBackground} \vspace{-7pt} {\parbox{\dimexpr\textwidth-2\fboxsep\relax}{\noindent \hspace*{-6pt}\includegraphics[width=5.8cm]{/web/www.cheatography.com/public/images/cheatography_logo.pdf}} } \end{tabulary} \columnbreak \begin{tabulary}{11cm}{L} \vspace{-2pt}\large{\bf{\textcolor{DarkBackground}{\textrm{Flink Cheat Sheet}}}} \\ \normalsize{by \textcolor{DarkBackground}{mliafol} via \textcolor{DarkBackground}{\uline{cheatography.com/70309/cs/39328/}}} \end{tabulary} \end{multicols}} \fancyfoot[L]{ \footnotesize \noindent \begin{multicols}{3} \begin{tabulary}{5.8cm}{LL} \SetRowColor{FootBackground} \mymulticolumn{2}{p{5.377cm}}{\bf\textcolor{white}{Cheatographer}} \\ \vspace{-2pt}mliafol \\ \uline{cheatography.com/mliafol} \\ \end{tabulary} \vfill \columnbreak \begin{tabulary}{5.8cm}{L} \SetRowColor{FootBackground} \mymulticolumn{1}{p{5.377cm}}{\bf\textcolor{white}{Cheat Sheet}} \\ \vspace{-2pt}Not Yet Published.\\ Updated 29th November, 2023.\\ Page {\thepage} of \pageref{LastPage}. \end{tabulary} \vfill \columnbreak \begin{tabulary}{5.8cm}{L} \SetRowColor{FootBackground} \mymulticolumn{1}{p{5.377cm}}{\bf\textcolor{white}{Sponsor}} \\ \SetRowColor{white} \vspace{-5pt} %\includegraphics[width=48px,height=48px]{dave.jpeg} Measure your website readability!\\ www.readability-score.com \end{tabulary} \end{multicols}} \begin{document} \raggedright \raggedcolumns % Set font size to small. Switch to any value % from this page to resize cheat sheet text: % www.emerson.emory.edu/services/latex/latex_169.html \footnotesize % Small font. \begin{tabularx}{17.67cm}{x{8.635 cm} x{8.635 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{Flink's API}} \tn % Row 0 \SetRowColor{LightBackground} SQL & Highlevel Language \tn % Row Count 1 (+ 1) % Row 1 \SetRowColor{white} Table API & Declarative DSL \tn % Row Count 2 (+ 1) % Row 2 \SetRowColor{LightBackground} DataStream/DataSet API & Core API \tn % Row Count 4 (+ 2) % Row 3 \SetRowColor{white} Stateful Stream Processing & Lower level building block \tn % Row Count 6 (+ 2) \hhline{>{\arrayrulecolor{DarkBackground}}--} \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{7.2534 cm} x{10.0166 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{DataSream}} \tn % Row 0 \SetRowColor{LightBackground} Data Stream & Immutable collections of data that can contain duplicates, can either be finite or unbounded \tn % Row Count 4 (+ 4) % Row 1 \SetRowColor{white} Flink program & Obtain en execution environment \tn % Row Count 6 (+ 2) % Row 2 \SetRowColor{LightBackground} & Load/create the initial data \tn % Row Count 8 (+ 2) % Row 3 \SetRowColor{white} & Transformation \tn % Row Count 9 (+ 1) % Row 4 \SetRowColor{LightBackground} & Where to put the result \tn % Row Count 10 (+ 1) % Row 5 \SetRowColor{white} & Trigger the execution \tn % Row Count 11 (+ 1) % Row 6 \SetRowColor{LightBackground} Flink program executed lazily & do not happen directly. Rather, operation is created and added to dataflow graph \tn % Row Count 15 (+ 4) \hhline{>{\arrayrulecolor{DarkBackground}}--} \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{4.2175 cm} x{6.2419 cm} x{6.4106 cm} } \SetRowColor{DarkBackground} \mymulticolumn{3}{x{17.67cm}}{\bf\textcolor{white}{Datasource Overview}} \tn % Row 0 \SetRowColor{LightBackground} \seqsplit{StreamExecutionEnvironment} & \seqsplit{getExecutionEnvironment();} & \tn % Row Count 3 (+ 3) % Row 1 \SetRowColor{white} Filebase \seqsplit{datasources} & \seqsplit{env.readFile(fileInputFormat}, path, watchType, interval, pathFilter, typeInfo) & watchType: can be \seqsplit{ileProcessingMode}.PROCESS\_CONTINUOUSLY or \seqsplit{FileProcessingMode}.PROCESS\_ONCE \tn % Row Count 10 (+ 7) % Row 2 \SetRowColor{LightBackground} \seqsplit{Socket-based:} & \seqsplit{env.socketTextStream} & \tn % Row Count 12 (+ 2) % Row 3 \SetRowColor{white} \seqsplit{Collection} based & env. \seqsplit{fromCollection}, \seqsplit{env.fromElements} & \tn % Row Count 15 (+ 3) % Row 4 \SetRowColor{LightBackground} Custom source & env.addSource & \tn % Row Count 17 (+ 2) % Row 5 \SetRowColor{white} A sequence numbers & \seqsplit{env.generateSequence(0}, 1000) & \tn % Row Count 20 (+ 3) \hhline{>{\arrayrulecolor{DarkBackground}}---} \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{8.635 cm} x{8.635 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{Data sink overview}} \tn % Row 0 \SetRowColor{LightBackground} writeAsText() / TextOutputFormat & Writes elements line-wise as Strings. \tn % Row Count 2 (+ 2) % Row 1 \SetRowColor{white} writeAsCsv() / CsvOutputFormat & Writes tuples as comma-separated value files. \tn % Row Count 5 (+ 3) % Row 2 \SetRowColor{LightBackground} \mymulticolumn{2}{x{17.67cm}}{print() / printToErr()} \tn % Row Count 6 (+ 1) % Row 3 \SetRowColor{white} \seqsplit{writeUsingOutputFormat()} / FileOutputFormat & Method and base class for custom file outputs \tn % Row Count 9 (+ 3) % Row 4 \SetRowColor{LightBackground} \mymulticolumn{2}{x{17.67cm}}{writeToSocket} \tn % Row Count 10 (+ 1) % Row 5 \SetRowColor{white} \mymulticolumn{2}{x{17.67cm}}{addSink} \tn % Row Count 11 (+ 1) \hhline{>{\arrayrulecolor{DarkBackground}}--} \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{8.635 cm} x{8.635 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{Timely Stream processing}} \tn % Row 0 \SetRowColor{LightBackground} Processing time & {\bf{System time of the machine that is executing the respective operation}} \tn % Row Count 4 (+ 4) % Row 1 \SetRowColor{white} & No coordination between streams and machines \tn % Row Count 7 (+ 3) % Row 2 \SetRowColor{LightBackground} & Best performance and lowest latency \tn % Row Count 9 (+ 2) % Row 3 \SetRowColor{white} & Not provide determinism in distributed and async enviroments \tn % Row Count 12 (+ 3) % Row 4 \SetRowColor{LightBackground} Event time & {\bf{The time that each individual event occurred on its producing device}} \tn % Row Count 16 (+ 4) % Row 5 \SetRowColor{white} & Extract from the records \tn % Row Count 18 (+ 2) % Row 6 \SetRowColor{LightBackground} & Consistent and deterministic \tn % Row Count 20 (+ 2) % Row 7 \SetRowColor{white} & High latency while waiting for out-of-order events \tn % Row Count 23 (+ 3) % Row 8 \SetRowColor{LightBackground} Watermark & {\bf{A mechanism to measure progress in event time}} \tn % Row Count 26 (+ 3) % Row 9 \SetRowColor{white} & Flow as part of the data stream and carry a timestamp t \tn % Row Count 29 (+ 3) % Row 10 \SetRowColor{LightBackground} & Watermark(t) declares that event time has reached time t, there should be no more elements with timestamp \textless{}= t \tn % Row Count 35 (+ 6) \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{8.635 cm} x{8.635 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{Timely Stream processing (cont)}} \tn % Row 11 \SetRowColor{LightBackground} & Crucial for out-of-order streams \tn % Row Count 2 (+ 2) % Row 12 \SetRowColor{white} Watermark strategy & TimestampAssigner + WatermarkGenerator \tn % Row Count 4 (+ 2) % Row 13 \SetRowColor{LightBackground} WatermarkGenerator & onEvent: Called for every event \tn % Row Count 6 (+ 2) % Row 14 \SetRowColor{white} & onPeriodicEmit: call periodically, and might emit a new watermark or not \tn % Row Count 10 (+ 4) % Row 15 \SetRowColor{LightBackground} & punctuate or periodic \tn % Row Count 12 (+ 2) % Row 16 \SetRowColor{white} \seqsplit{WatermarkStrategy.forMonotonousTimestamps();} & Event time itself \tn % Row Count 15 (+ 3) % Row 17 \SetRowColor{LightBackground} \seqsplit{WatermarkStrategy.forBoundedOutOfOrderness} & Watermark lags behind the maximum timestamp seen in the stream by a fixed amount of time \tn % Row Count 20 (+ 5) \hhline{>{\arrayrulecolor{DarkBackground}}--} \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{6.3899 cm} x{10.8801 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{State}} \tn % Row 0 \SetRowColor{LightBackground} Stateful operator & Remember information acc \tn % Row Count 2 (+ 2) % Row 1 \SetRowColor{white} Keyed state & Embedded key/value store \tn % Row Count 3 (+ 1) % Row 2 \SetRowColor{LightBackground} & Partitioned and distributed strictly together with the streams \tn % Row Count 6 (+ 3) % Row 3 \SetRowColor{white} & Only on keyed stream \tn % Row Count 7 (+ 1) % Row 4 \SetRowColor{LightBackground} State persistence & Fault tolerance: stream replay and checkpointing \tn % Row Count 9 (+ 2) % Row 5 \SetRowColor{white} Checkpoint & Marks a specific point in each of the input streams along with the corresponding state for each operators \tn % Row Count 14 (+ 5) % Row 6 \SetRowColor{LightBackground} & Drawing consistent snapshots of the distributed data stream and operator state \tn % Row Count 18 (+ 4) % Row 7 \SetRowColor{white} Stream barriers & Injected into the data stream and flow with the records as part of the data stream \tn % Row Count 22 (+ 4) % Row 8 \SetRowColor{LightBackground} & Separated the records in the data stream into the set of records that goes into the current snapshot, and the records that go into the next snapshot. \tn % Row Count 28 (+ 6) % Row 9 \SetRowColor{white} & The point where the barriers for snapshot n are injected, is the position in the source stream up to which snapshot cover the data \tn % Row Count 34 (+ 6) \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{6.3899 cm} x{10.8801 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{State (cont)}} \tn % Row 10 \SetRowColor{LightBackground} & Alignment phrase: Receive barrier for snapshot n of one incoming stream, operator need to wait until receive all others input \tn % Row Count 5 (+ 5) % Row 11 \SetRowColor{white} Snapshot operator state & At the point in time when they received all barriers from input streams and before emitting the barriers to their output streams \tn % Row Count 11 (+ 6) % Row 12 \SetRowColor{LightBackground} & For each parallel stream data source, the offset/position in the stream when the snapshot started \tn % Row Count 15 (+ 4) % Row 13 \SetRowColor{white} & For each operator, a pointer to the state that was stored \tn % Row Count 18 (+ 3) % Row 14 \SetRowColor{LightBackground} Unaligned checkpoint & Reacts on the first barrier that is stored in its input buffers \tn % Row Count 21 (+ 3) % Row 15 \SetRowColor{white} Checkpoint & Simple external dependencies \tn % Row Count 23 (+ 2) % Row 16 \SetRowColor{LightBackground} & Immutable and versioned \tn % Row Count 24 (+ 1) % Row 17 \SetRowColor{white} & Decouple the stream transport from the persistence mechanism \tn % Row Count 27 (+ 3) % Row 18 \SetRowColor{LightBackground} Backpressure & Slow receiver makes the senders slow down in order not to overwhelm the receiver \tn % Row Count 31 (+ 4) \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{6.3899 cm} x{10.8801 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{State (cont)}} \tn % Row 19 \SetRowColor{LightBackground} Snapshot & generic term refer to global, consistent image of a state of a Flink job \tn % Row Count 3 (+ 3) \hhline{>{\arrayrulecolor{DarkBackground}}--} \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{X} \SetRowColor{DarkBackground} \mymulticolumn{1}{x{17.67cm}}{\bf\textcolor{white}{Flink architecture}} \tn \SetRowColor{LightBackground} \mymulticolumn{1}{p{17.67cm}}{\vspace{1px}\centerline{\includegraphics[width=5.1cm]{/web/www.cheatography.com/public/uploads/mliafol_1697183640_Screenshot 2023-10-13 at 3.53.21 PM.png}}} \tn \hhline{>{\arrayrulecolor{DarkBackground}}-} \SetRowColor{LightBackground} \mymulticolumn{1}{x{17.67cm}}{The client is not part of the runtime and program execution, but is used to prepare and send a dataflow to the JobManager. After that, client can disconnect(detached mode), or stay connected (attached mode)} \tn \hhline{>{\arrayrulecolor{DarkBackground}}-} \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{6.3899 cm} x{10.8801 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{RocksDB tunning}} \tn % Row 0 \SetRowColor{LightBackground} Incremental checkpoints & Record the changes compared to the previous completed checkpoint, instead of producing a full, self-contained backup \tn % Row Count 5 (+ 5) % Row 1 \SetRowColor{white} Timers & Schedule actions for later =\textgreater{} save on healp =\textgreater{} \tn % Row Count 7 (+ 2) % Row 2 \SetRowColor{LightBackground} & \seqsplit{state.backend.rocksdb.timer-service.factory} =heap \tn % Row Count 9 (+ 2) % Row 3 \SetRowColor{white} Tunning rocksdb memory & Flink's managed memory to buffer and cache \tn % Row Count 11 (+ 2) % Row 4 \SetRowColor{LightBackground} & Increase the amount of managed memory \tn % Row Count 13 (+ 2) \hhline{>{\arrayrulecolor{DarkBackground}}--} \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{5.8718 cm} x{11.3982 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{Window}} \tn % Row 0 \SetRowColor{LightBackground} {\bf{Definition}} & Split stream into buckets of finite size, over which we can apply computations \tn % Row Count 3 (+ 3) % Row 1 \SetRowColor{white} {\bf{Keyed windows}} & .keyBy().window().{[}.triger(){]}{[}.evictor(){]}{[}.allowedLateness(){]}{[}.sideOutputLateData(){]}.reduce/aggregate/apply \tn % Row Count 8 (+ 5) % Row 2 \SetRowColor{LightBackground} & Be performed in parallel by multiple tasks \tn % Row Count 10 (+ 2) % Row 3 \SetRowColor{white} {\bf{Non-Keyed windows}} & windowAll().{[}.triger(){]}{[}.evictor(){]}{[}.allowedLateness(){]}{[}.sideOutputLateData(){]}.reduce/aggregate/apply \tn % Row Count 14 (+ 4) % Row 4 \SetRowColor{LightBackground} & Be performed by a single task (parallelism = 1) \tn % Row Count 16 (+ 2) % Row 5 \SetRowColor{white} {\bf{Lifecycle}} & Created : the first element belong to this window arrie \tn % Row Count 19 (+ 3) % Row 6 \SetRowColor{LightBackground} & Removed: the time passes its end timestamp + allowed lateness \tn % Row Count 22 (+ 3) % Row 7 \SetRowColor{white} {\bf{Window Assigner}} & Responsible for assigning each incoming element to 1 or more windows \tn % Row Count 25 (+ 3) % Row 8 \SetRowColor{LightBackground} & Assign based on time: start timestamp (inclusive) and an end timestime(exclusive) \tn % Row Count 29 (+ 4) % Row 9 \SetRowColor{white} & TumblingWindows: each element to {\bf{a window}} of a specified window size. {\bf{Fixed size and not overlap}} \tn % Row Count 33 (+ 4) \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{5.8718 cm} x{11.3982 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{Window (cont)}} \tn % Row 10 \SetRowColor{LightBackground} & SlidingWindows: each element to {\bf{windows}}. {\bf{Fixed size and can be overlapping }}(window slide \textless{} window size) \tn % Row Count 5 (+ 5) % Row 11 \SetRowColor{white} & SessionWindows: assigner groups elements by {\bf{sessions of activity}}. Dont overlap, dont have fixed time. Close when it does not receive elements for a certain period of time \tn % Row Count 12 (+ 7) % Row 12 \SetRowColor{LightBackground} & GlobalWindows: all elements with the same key to {\bf{same global window}}. Only useful if specify a custom trigger, because it does not have a natural end \tn % Row Count 18 (+ 6) % Row 13 \SetRowColor{white} {\bf{Window Functions}} & Computation that perform on each of windows \tn % Row Count 20 (+ 2) % Row 14 \SetRowColor{LightBackground} \seqsplit{ReduceFunction} & Incrementally aggregate \tn % Row Count 22 (+ 2) % Row 15 \SetRowColor{white} & Two elements from the input are combined to produce an output element with the same type \tn % Row Count 26 (+ 4) % Row 16 \SetRowColor{LightBackground} \seqsplit{AggregateFunction} & Generalised version of a ReduceFunction with 3 types: IN, ACC, OUT \tn % Row Count 29 (+ 3) % Row 17 \SetRowColor{white} & Methods: creating initial accumulator, merging, extract output \tn % Row Count 32 (+ 3) \end{tabularx} \par\addvspace{1.3em} \begin{tabularx}{17.67cm}{x{5.8718 cm} x{11.3982 cm} } \SetRowColor{DarkBackground} \mymulticolumn{2}{x{17.67cm}}{\bf\textcolor{white}{Window (cont)}} \tn % Row 18 \SetRowColor{LightBackground} \seqsplit{ProcessWindowFunction} & Iterable containing all the elements of the window \tn % Row Count 2 (+ 2) % Row 19 \SetRowColor{white} & Context object with time and state information \tn % Row Count 4 (+ 2) \hhline{>{\arrayrulecolor{DarkBackground}}--} \end{tabularx} \par\addvspace{1.3em} \end{document}