diff --git a/isleOfWight/cleaningFeederData.Rmd b/isleOfWight/cleaningFeederData.Rmd index fea71533455c81b372c75abd9efa6515d08b73c9..a07786ef260987b189b0dcac42ea5fb8a3f4a748 100644 --- a/isleOfWight/cleaningFeederData.Rmd +++ b/isleOfWight/cleaningFeederData.Rmd @@ -4,24 +4,24 @@ subtitle: 'Code and notes' author: "Ben Anderson (b.anderson@soton.ac.uk), [SERG, Energy & Climate Change](http://www.energy.soton.ac.uk/), University of Southampton" date: 'Last run at: `r Sys.time()`' output: - bookdown::html_document2: - code_folding: hide + bookdown::pdf_document2: fig_caption: yes + keep_tex: yes number_sections: yes - self_contained: yes toc: yes - toc_depth: 3 - toc_float: yes + toc_depth: 2 bookdown::word_document2: fig_caption: yes toc: yes toc_depth: 2 - bookdown::pdf_document2: + bookdown::html_document2: + code_folding: hide fig_caption: yes - keep_tex: yes number_sections: yes + self_contained: yes toc: yes - toc_depth: 2 + toc_depth: 3 + toc_float: yes always_allow_html: yes bibliography: '`r path.expand("~/bibliography.bib")`' --- @@ -94,7 +94,7 @@ head(dt) Do some data prep -```{r skimIt} +```{r dataSummary} dt[, rDateTime := lubridate::as_datetime(DateTime)] dt[, rTime := hms::as_hms(rDateTime)] @@ -105,7 +105,11 @@ dt[, kW := Value] # for clarity dt[, Value := NULL] #drop dt <- addSeason(dt, dateVar = "rDate", h = "N") -skimr::skim(dt) +#skimr::skim(dt) doesn't play nicely with latex +st <- summary(dt[,.(rYear, rDateTime, sub_region, kW, rDoW, season)]) +kableExtra::kable(st, + caption = "Summary of loaded and processed data") %>% + kable_styling() ``` Do a sense check by season & feeder. @@ -241,7 +245,6 @@ Analysis completed in `r round(elapsed,2)` seconds ( `r round(elapsed/60,2)` min * lubridate [@lubridate] * rmarkdown [@rmarkdown] * skimr [@skimr] - * XML [@XML] ## Session info diff --git a/isleOfWight/cleaningFeederData.log b/isleOfWight/cleaningFeederData.log new file mode 100644 index 0000000000000000000000000000000000000000..56e3706ca40f810199acdac3e9c5cb568713e4de --- /dev/null +++ b/isleOfWight/cleaningFeederData.log @@ -0,0 +1,1180 @@ +This is pdfTeX, Version 3.14159265-2.6-1.40.20 (TeX Live 2019) (preloaded format=pdflatex 2019.5.23) 7 JUL 2020 17:25 +entering extended mode + restricted \write18 enabled. + %&-line parsing enabled. +**cleaningFeederData.tex +(./cleaningFeederData.tex +LaTeX2e <2018-12-01> +(/usr/local/texlive/2019/texmf-dist/tex/latex/base/article.cls +Document Class: article 2018/09/03 v1.4i Standard LaTeX document class +(/usr/local/texlive/2019/texmf-dist/tex/latex/base/size10.clo +File: size10.clo 2018/09/03 v1.4i Standard LaTeX file (size option) +) +\c@part=\count80 +\c@section=\count81 +\c@subsection=\count82 +\c@subsubsection=\count83 +\c@paragraph=\count84 +\c@subparagraph=\count85 +\c@figure=\count86 +\c@table=\count87 +\abovecaptionskip=\skip41 +\belowcaptionskip=\skip42 +\bibindent=\dimen102 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/lm/lmodern.sty +Package: lmodern 2009/10/30 v1.6 Latin Modern Fonts +LaTeX Font Info: Overwriting symbol font `operators' in version `normal' +(Font) OT1/cmr/m/n --> OT1/lmr/m/n on input line 22. +LaTeX Font Info: Overwriting symbol font `letters' in version `normal' +(Font) OML/cmm/m/it --> OML/lmm/m/it on input line 23. +LaTeX Font Info: Overwriting symbol font `symbols' in version `normal' +(Font) OMS/cmsy/m/n --> OMS/lmsy/m/n on input line 24. +LaTeX Font Info: Overwriting symbol font `largesymbols' in version `normal' +(Font) OMX/cmex/m/n --> OMX/lmex/m/n on input line 25. +LaTeX Font Info: Overwriting symbol font `operators' in version `bold' +(Font) OT1/cmr/bx/n --> OT1/lmr/bx/n on input line 26. +LaTeX Font Info: Overwriting symbol font `letters' in version `bold' +(Font) OML/cmm/b/it --> OML/lmm/b/it on input line 27. +LaTeX Font Info: Overwriting symbol font `symbols' in version `bold' +(Font) OMS/cmsy/b/n --> OMS/lmsy/b/n on input line 28. +LaTeX Font Info: Overwriting symbol font `largesymbols' in version `bold' +(Font) OMX/cmex/m/n --> OMX/lmex/m/n on input line 29. +LaTeX Font Info: Overwriting math alphabet `\mathbf' in version `normal' +(Font) OT1/cmr/bx/n --> OT1/lmr/bx/n on input line 31. +LaTeX Font Info: Overwriting math alphabet `\mathsf' in version `normal' +(Font) OT1/cmss/m/n --> OT1/lmss/m/n on input line 32. +LaTeX Font Info: Overwriting math alphabet `\mathit' in version `normal' +(Font) OT1/cmr/m/it --> OT1/lmr/m/it on input line 33. +LaTeX Font Info: Overwriting math alphabet `\mathtt' in version `normal' +(Font) OT1/cmtt/m/n --> OT1/lmtt/m/n on input line 34. +LaTeX Font Info: Overwriting math alphabet `\mathbf' in version `bold' +(Font) OT1/cmr/bx/n --> OT1/lmr/bx/n on input line 35. +LaTeX Font Info: Overwriting math alphabet `\mathsf' in version `bold' +(Font) OT1/cmss/bx/n --> OT1/lmss/bx/n on input line 36. +LaTeX Font Info: Overwriting math alphabet `\mathit' in version `bold' +(Font) OT1/cmr/bx/it --> OT1/lmr/bx/it on input line 37. +LaTeX Font Info: Overwriting math alphabet `\mathtt' in version `bold' +(Font) OT1/cmtt/m/n --> OT1/lmtt/m/n on input line 38. +) (/usr/local/texlive/2019/texmf-dist/tex/latex/amsfonts/amssymb.sty +Package: amssymb 2013/01/14 v3.01 AMS font symbols +(/usr/local/texlive/2019/texmf-dist/tex/latex/amsfonts/amsfonts.sty +Package: amsfonts 2013/01/14 v3.01 Basic AMSFonts support +\@emptytoks=\toks14 +\symAMSa=\mathgroup4 +\symAMSb=\mathgroup5 +LaTeX Font Info: Overwriting math alphabet `\mathfrak' in version `bold' +(Font) U/euf/m/n --> U/euf/b/n on input line 106. +)) (/usr/local/texlive/2019/texmf-dist/tex/latex/amsmath/amsmath.sty +Package: amsmath 2018/12/01 v2.17b AMS math features +\@mathmargin=\skip43 +For additional information on amsmath, use the `?' option. +(/usr/local/texlive/2019/texmf-dist/tex/latex/amsmath/amstext.sty +Package: amstext 2000/06/29 v2.01 AMS text +(/usr/local/texlive/2019/texmf-dist/tex/latex/amsmath/amsgen.sty +File: amsgen.sty 1999/11/30 v2.0 generic functions +\@emptytoks=\toks15 +\ex@=\dimen103 +)) (/usr/local/texlive/2019/texmf-dist/tex/latex/amsmath/amsbsy.sty +Package: amsbsy 1999/11/29 v1.2d Bold Symbols +\pmbraise@=\dimen104 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/amsmath/amsopn.sty +Package: amsopn 2016/03/08 v2.02 operator names +) +\inf@bad=\count88 +LaTeX Info: Redefining \frac on input line 223. +\uproot@=\count89 +\leftroot@=\count90 +LaTeX Info: Redefining \overline on input line 385. +\classnum@=\count91 +\DOTSCASE@=\count92 +LaTeX Info: Redefining \ldots on input line 482. +LaTeX Info: Redefining \dots on input line 485. +LaTeX Info: Redefining \cdots on input line 606. +\Mathstrutbox@=\box27 +\strutbox@=\box28 +\big@size=\dimen105 +LaTeX Font Info: Redeclaring font encoding OML on input line 729. +LaTeX Font Info: Redeclaring font encoding OMS on input line 730. +\macc@depth=\count93 +\c@MaxMatrixCols=\count94 +\dotsspace@=\muskip10 +\c@parentequation=\count95 +\dspbrk@lvl=\count96 +\tag@help=\toks16 +\row@=\count97 +\column@=\count98 +\maxfields@=\count99 +\andhelp@=\toks17 +\eqnshift@=\dimen106 +\alignsep@=\dimen107 +\tagshift@=\dimen108 +\tagwidth@=\dimen109 +\totwidth@=\dimen110 +\lineht@=\dimen111 +\@envbody=\toks18 +\multlinegap=\skip44 +\multlinetaggap=\skip45 +\mathdisplay@stack=\toks19 +LaTeX Info: Redefining \[ on input line 2844. +LaTeX Info: Redefining \] on input line 2845. +) (/usr/local/texlive/2019/texmf-dist/tex/generic/ifxetex/ifxetex.sty +Package: ifxetex 2010/09/12 v0.6 Provides ifxetex conditional +) (/usr/local/texlive/2019/texmf-dist/tex/generic/oberdiek/ifluatex.sty +Package: ifluatex 2016/05/16 v1.4 Provides the ifluatex switch (HO) +Package ifluatex Info: LuaTeX not detected. +) (/usr/local/texlive/2019/texmf-dist/tex/latex/base/fixltx2e.sty +Package: fixltx2e 2016/12/29 v2.1a fixes to LaTeX (obsolete) +Applying: [2015/01/01] Old fixltx2e package on input line 46. + +Package fixltx2e Warning: fixltx2e is not required with releases after 2015 +(fixltx2e) All fixes are now in the LaTeX kernel. +(fixltx2e) See the latexrelease package for details. + +Already applied: [0000/00/00] Old fixltx2e package on input line 53. +) (/usr/local/texlive/2019/texmf-dist/tex/latex/base/fontenc.sty +Package: fontenc 2018/08/11 v2.0j Standard LaTeX package +(/usr/local/texlive/2019/texmf-dist/tex/latex/base/t1enc.def +File: t1enc.def 2018/08/11 v2.0j Standard LaTeX file +LaTeX Font Info: Redeclaring font encoding T1 on input line 48. +)) (/usr/local/texlive/2019/texmf-dist/tex/latex/base/inputenc.sty +Package: inputenc 2018/08/11 v1.3c Input encoding file +\inpenc@prehook=\toks20 +\inpenc@posthook=\toks21 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/base/textcomp.sty +Package: textcomp 2018/08/11 v2.0j Standard LaTeX package +Package textcomp Info: Sub-encoding information: +(textcomp) 5 = only ISO-Adobe without \textcurrency +(textcomp) 4 = 5 + \texteuro +(textcomp) 3 = 4 + \textohm +(textcomp) 2 = 3 + \textestimated + \textcurrency +(textcomp) 1 = TS1 - \textcircled - \t +(textcomp) 0 = TS1 (full) +(textcomp) Font families with sub-encoding setting implement +(textcomp) only a restricted character set as indicated. +(textcomp) Family '?' is the default used for unknown fonts. +(textcomp) See the documentation for details. +Package textcomp Info: Setting ? sub-encoding to TS1/1 on input line 79. +(/usr/local/texlive/2019/texmf-dist/tex/latex/base/ts1enc.def +File: ts1enc.def 2001/06/05 v3.0e (jk/car/fm) Standard LaTeX file +Now handling font encoding TS1 ... +... processing UTF-8 mapping file for font encoding TS1 +(/usr/local/texlive/2019/texmf-dist/tex/latex/base/ts1enc.dfu +File: ts1enc.dfu 2018/10/05 v1.2f UTF-8 support for inputenc + defining Unicode char U+00A2 (decimal 162) + defining Unicode char U+00A3 (decimal 163) + defining Unicode char U+00A4 (decimal 164) + defining Unicode char U+00A5 (decimal 165) + defining Unicode char U+00A6 (decimal 166) + defining Unicode char U+00A7 (decimal 167) + defining Unicode char U+00A8 (decimal 168) + defining Unicode char U+00A9 (decimal 169) + defining Unicode char U+00AA (decimal 170) + defining Unicode char U+00AC (decimal 172) + defining Unicode char U+00AE (decimal 174) + defining Unicode char U+00AF (decimal 175) + defining Unicode char U+00B0 (decimal 176) + defining Unicode char U+00B1 (decimal 177) + defining Unicode char U+00B2 (decimal 178) + defining Unicode char U+00B3 (decimal 179) + defining Unicode char U+00B4 (decimal 180) + defining Unicode char U+00B5 (decimal 181) + defining Unicode char U+00B6 (decimal 182) + defining Unicode char U+00B7 (decimal 183) + defining Unicode char U+00B9 (decimal 185) + defining Unicode char U+00BA (decimal 186) + defining Unicode char U+00BC (decimal 188) + defining Unicode char U+00BD (decimal 189) + defining Unicode char U+00BE (decimal 190) + defining Unicode char U+00D7 (decimal 215) + defining Unicode char U+00F7 (decimal 247) + defining Unicode char U+0192 (decimal 402) + defining Unicode char U+02C7 (decimal 711) + defining Unicode char U+02D8 (decimal 728) + defining Unicode char U+02DD (decimal 733) + defining Unicode char U+0E3F (decimal 3647) + defining Unicode char U+2016 (decimal 8214) + defining Unicode char U+2020 (decimal 8224) + defining Unicode char U+2021 (decimal 8225) + defining Unicode char U+2022 (decimal 8226) + defining Unicode char U+2030 (decimal 8240) + defining Unicode char U+2031 (decimal 8241) + defining Unicode char U+203B (decimal 8251) + defining Unicode char U+203D (decimal 8253) + defining Unicode char U+2044 (decimal 8260) + defining Unicode char U+204E (decimal 8270) + defining Unicode char U+2052 (decimal 8274) + defining Unicode char U+20A1 (decimal 8353) + defining Unicode char U+20A4 (decimal 8356) + defining Unicode char U+20A6 (decimal 8358) + defining Unicode char U+20A9 (decimal 8361) + defining Unicode char U+20AB (decimal 8363) + defining Unicode char U+20AC (decimal 8364) + defining Unicode char U+20B1 (decimal 8369) + defining Unicode char U+2103 (decimal 8451) + defining Unicode char U+2116 (decimal 8470) + defining Unicode char U+2117 (decimal 8471) + defining Unicode char U+211E (decimal 8478) + defining Unicode char U+2120 (decimal 8480) + defining Unicode char U+2122 (decimal 8482) + defining Unicode char U+2126 (decimal 8486) + defining Unicode char U+2127 (decimal 8487) + defining Unicode char U+212E (decimal 8494) + defining Unicode char U+2190 (decimal 8592) + defining Unicode char U+2191 (decimal 8593) + defining Unicode char U+2192 (decimal 8594) + defining Unicode char U+2193 (decimal 8595) + defining Unicode char U+2329 (decimal 9001) + defining Unicode char U+232A (decimal 9002) + defining Unicode char U+2422 (decimal 9250) + defining Unicode char U+25E6 (decimal 9702) + defining Unicode char U+25EF (decimal 9711) + defining Unicode char U+266A (decimal 9834) + defining Unicode char U+FEFF (decimal 65279) +)) +LaTeX Info: Redefining \oldstylenums on input line 334. +Package textcomp Info: Setting cmr sub-encoding to TS1/0 on input line 349. +Package textcomp Info: Setting cmss sub-encoding to TS1/0 on input line 350. +Package textcomp Info: Setting cmtt sub-encoding to TS1/0 on input line 351. +Package textcomp Info: Setting cmvtt sub-encoding to TS1/0 on input line 352. +Package textcomp Info: Setting cmbr sub-encoding to TS1/0 on input line 353. +Package textcomp Info: Setting cmtl sub-encoding to TS1/0 on input line 354. +Package textcomp Info: Setting ccr sub-encoding to TS1/0 on input line 355. +Package textcomp Info: Setting ptm sub-encoding to TS1/4 on input line 356. +Package textcomp Info: Setting pcr sub-encoding to TS1/4 on input line 357. +Package textcomp Info: Setting phv sub-encoding to TS1/4 on input line 358. +Package textcomp Info: Setting ppl sub-encoding to TS1/3 on input line 359. +Package textcomp Info: Setting pag sub-encoding to TS1/4 on input line 360. +Package textcomp Info: Setting pbk sub-encoding to TS1/4 on input line 361. +Package textcomp Info: Setting pnc sub-encoding to TS1/4 on input line 362. +Package textcomp Info: Setting pzc sub-encoding to TS1/4 on input line 363. +Package textcomp Info: Setting bch sub-encoding to TS1/4 on input line 364. +Package textcomp Info: Setting put sub-encoding to TS1/5 on input line 365. +Package textcomp Info: Setting uag sub-encoding to TS1/5 on input line 366. +Package textcomp Info: Setting ugq sub-encoding to TS1/5 on input line 367. +Package textcomp Info: Setting ul8 sub-encoding to TS1/4 on input line 368. +Package textcomp Info: Setting ul9 sub-encoding to TS1/4 on input line 369. +Package textcomp Info: Setting augie sub-encoding to TS1/5 on input line 370. +Package textcomp Info: Setting dayrom sub-encoding to TS1/3 on input line 371. +Package textcomp Info: Setting dayroms sub-encoding to TS1/3 on input line 372. + +Package textcomp Info: Setting pxr sub-encoding to TS1/0 on input line 373. +Package textcomp Info: Setting pxss sub-encoding to TS1/0 on input line 374. +Package textcomp Info: Setting pxtt sub-encoding to TS1/0 on input line 375. +Package textcomp Info: Setting txr sub-encoding to TS1/0 on input line 376. +Package textcomp Info: Setting txss sub-encoding to TS1/0 on input line 377. +Package textcomp Info: Setting txtt sub-encoding to TS1/0 on input line 378. +Package textcomp Info: Setting lmr sub-encoding to TS1/0 on input line 379. +Package textcomp Info: Setting lmdh sub-encoding to TS1/0 on input line 380. +Package textcomp Info: Setting lmss sub-encoding to TS1/0 on input line 381. +Package textcomp Info: Setting lmssq sub-encoding to TS1/0 on input line 382. +Package textcomp Info: Setting lmvtt sub-encoding to TS1/0 on input line 383. +Package textcomp Info: Setting lmtt sub-encoding to TS1/0 on input line 384. +Package textcomp Info: Setting qhv sub-encoding to TS1/0 on input line 385. +Package textcomp Info: Setting qag sub-encoding to TS1/0 on input line 386. +Package textcomp Info: Setting qbk sub-encoding to TS1/0 on input line 387. +Package textcomp Info: Setting qcr sub-encoding to TS1/0 on input line 388. +Package textcomp Info: Setting qcs sub-encoding to TS1/0 on input line 389. +Package textcomp Info: Setting qpl sub-encoding to TS1/0 on input line 390. +Package textcomp Info: Setting qtm sub-encoding to TS1/0 on input line 391. +Package textcomp Info: Setting qzc sub-encoding to TS1/0 on input line 392. +Package textcomp Info: Setting qhvc sub-encoding to TS1/0 on input line 393. +Package textcomp Info: Setting futs sub-encoding to TS1/4 on input line 394. +Package textcomp Info: Setting futx sub-encoding to TS1/4 on input line 395. +Package textcomp Info: Setting futj sub-encoding to TS1/4 on input line 396. +Package textcomp Info: Setting hlh sub-encoding to TS1/3 on input line 397. +Package textcomp Info: Setting hls sub-encoding to TS1/3 on input line 398. +Package textcomp Info: Setting hlst sub-encoding to TS1/3 on input line 399. +Package textcomp Info: Setting hlct sub-encoding to TS1/5 on input line 400. +Package textcomp Info: Setting hlx sub-encoding to TS1/5 on input line 401. +Package textcomp Info: Setting hlce sub-encoding to TS1/5 on input line 402. +Package textcomp Info: Setting hlcn sub-encoding to TS1/5 on input line 403. +Package textcomp Info: Setting hlcw sub-encoding to TS1/5 on input line 404. +Package textcomp Info: Setting hlcf sub-encoding to TS1/5 on input line 405. +Package textcomp Info: Setting pplx sub-encoding to TS1/3 on input line 406. +Package textcomp Info: Setting pplj sub-encoding to TS1/3 on input line 407. +Package textcomp Info: Setting ptmx sub-encoding to TS1/4 on input line 408. +Package textcomp Info: Setting ptmj sub-encoding to TS1/4 on input line 409. +) (/usr/local/texlive/2019/texmf-dist/tex/latex/upquote/upquote.sty +Package: upquote 2012/04/19 v1.3 upright-quote and grave-accent glyphs in verba +tim +) (/usr/local/texlive/2019/texmf-dist/tex/latex/microtype/microtype.sty +Package: microtype 2019/02/28 v2.7b Micro-typographical refinements (RS) +(/usr/local/texlive/2019/texmf-dist/tex/latex/graphics/keyval.sty +Package: keyval 2014/10/28 v1.15 key=value parser (DPC) +\KV@toks@=\toks22 +) +\MT@toks=\toks23 +\MT@count=\count100 +LaTeX Info: Redefining \textls on input line 790. +\MT@outer@kern=\dimen112 +LaTeX Info: Redefining \textmicrotypecontext on input line 1336. +\MT@listname@count=\count101 +(/usr/local/texlive/2019/texmf-dist/tex/latex/microtype/microtype-pdftex.def +File: microtype-pdftex.def 2019/02/28 v2.7b Definitions specific to pdftex (RS) + +LaTeX Info: Redefining \lsstyle on input line 914. +LaTeX Info: Redefining \lslig on input line 914. +\MT@outer@space=\skip46 +) +Package microtype Info: Loading configuration file microtype.cfg. +(/usr/local/texlive/2019/texmf-dist/tex/latex/microtype/microtype.cfg +File: microtype.cfg 2019/02/28 v2.7b microtype main configuration file (RS) +)) (/usr/local/texlive/2019/texmf-dist/tex/latex/parskip/parskip.sty +Package: parskip 2019-01-16 v2.0c non-zero parskip adjustments +(/usr/local/texlive/2019/texmf-dist/tex/latex/oberdiek/kvoptions.sty +Package: kvoptions 2016/05/16 v3.12 Key value format for package options (HO) +(/usr/local/texlive/2019/texmf-dist/tex/generic/oberdiek/ltxcmds.sty +Package: ltxcmds 2016/05/16 v1.23 LaTeX kernel commands for general use (HO) +) (/usr/local/texlive/2019/texmf-dist/tex/generic/oberdiek/kvsetkeys.sty +Package: kvsetkeys 2016/05/16 v1.17 Key value parser (HO) +(/usr/local/texlive/2019/texmf-dist/tex/generic/oberdiek/infwarerr.sty +Package: infwarerr 2016/05/16 v1.4 Providing info/warning/error messages (HO) +) (/usr/local/texlive/2019/texmf-dist/tex/generic/oberdiek/etexcmds.sty +Package: etexcmds 2016/05/16 v1.6 Avoid name clashes with e-TeX commands (HO) +))) (/usr/local/texlive/2019/texmf-dist/tex/latex/etoolbox/etoolbox.sty +Package: etoolbox 2018/08/19 v2.5f e-TeX tools for LaTeX (JAW) +\etb@tempcnta=\count102 +)) (/usr/local/texlive/2019/texmf-dist/tex/latex/hyperref/hyperref.sty +Package: hyperref 2018/11/30 v6.88e Hypertext links for LaTeX +(/usr/local/texlive/2019/texmf-dist/tex/generic/oberdiek/hobsub-hyperref.sty +Package: hobsub-hyperref 2016/05/16 v1.14 Bundle oberdiek, subset hyperref (HO) + +(/usr/local/texlive/2019/texmf-dist/tex/generic/oberdiek/hobsub-generic.sty +Package: hobsub-generic 2016/05/16 v1.14 Bundle oberdiek, subset generic (HO) +Package: hobsub 2016/05/16 v1.14 Construct package bundles (HO) +Package hobsub Info: Skipping package `infwarerr' (already loaded). +Package hobsub Info: Skipping package `ltxcmds' (already loaded). +Package hobsub Info: Skipping package `ifluatex' (already loaded). +Package: ifvtex 2016/05/16 v1.6 Detect VTeX and its facilities (HO) +Package ifvtex Info: VTeX not detected. +Package: intcalc 2016/05/16 v1.2 Expandable calculations with integers (HO) +Package: ifpdf 2018/09/07 v3.3 Provides the ifpdf switch +Package hobsub Info: Skipping package `etexcmds' (already loaded). +Package hobsub Info: Skipping package `kvsetkeys' (already loaded). +Package: kvdefinekeys 2016/05/16 v1.4 Define keys (HO) +Package: pdftexcmds 2018/09/10 v0.29 Utility functions of pdfTeX for LuaTeX (HO +) +Package pdftexcmds Info: LuaTeX not detected. +Package pdftexcmds Info: \pdf@primitive is available. +Package pdftexcmds Info: \pdf@ifprimitive is available. +Package pdftexcmds Info: \pdfdraftmode found. +Package: pdfescape 2016/05/16 v1.14 Implements pdfTeX's escape features (HO) +Package: bigintcalc 2016/05/16 v1.4 Expandable calculations on big integers (HO +) +Package: bitset 2016/05/16 v1.2 Handle bit-vector datatype (HO) +Package: uniquecounter 2016/05/16 v1.3 Provide unlimited unique counter (HO) +) +Package hobsub Info: Skipping package `hobsub' (already loaded). +Package: letltxmacro 2016/05/16 v1.5 Let assignment for LaTeX macros (HO) +Package: hopatch 2016/05/16 v1.3 Wrapper for package hooks (HO) +Package: xcolor-patch 2016/05/16 xcolor patch +Package: atveryend 2016/05/16 v1.9 Hooks at the very end of document (HO) +Package: atbegshi 2016/06/09 v1.18 At begin shipout hook (HO) +Package: refcount 2016/05/16 v3.5 Data extraction from label references (HO) +Package: hycolor 2016/05/16 v1.8 Color options for hyperref/bookmark (HO) +) (/usr/local/texlive/2019/texmf-dist/tex/latex/oberdiek/auxhook.sty +Package: auxhook 2016/05/16 v1.4 Hooks for auxiliary files (HO) +) +\@linkdim=\dimen113 +\Hy@linkcounter=\count103 +\Hy@pagecounter=\count104 +(/usr/local/texlive/2019/texmf-dist/tex/latex/hyperref/pd1enc.def +File: pd1enc.def 2018/11/30 v6.88e Hyperref: PDFDocEncoding definition (HO) +Now handling font encoding PD1 ... +... no UTF-8 mapping file for font encoding PD1 +) +\Hy@SavedSpaceFactor=\count105 +(/usr/local/texlive/2019/texmf-dist/tex/latex/latexconfig/hyperref.cfg +File: hyperref.cfg 2002/06/06 v1.2 hyperref configuration of TeXLive +) +Package hyperref Info: Option `unicode' set `true' on input line 4393. +(/usr/local/texlive/2019/texmf-dist/tex/latex/hyperref/puenc.def +File: puenc.def 2018/11/30 v6.88e Hyperref: PDF Unicode definition (HO) +Now handling font encoding PU ... +... no UTF-8 mapping file for font encoding PU +) +Package hyperref Info: Hyper figures OFF on input line 4519. +Package hyperref Info: Link nesting OFF on input line 4524. +Package hyperref Info: Hyper index ON on input line 4527. +Package hyperref Info: Plain pages OFF on input line 4534. +Package hyperref Info: Backreferencing OFF on input line 4539. +Package hyperref Info: Implicit mode ON; LaTeX internals redefined. +Package hyperref Info: Bookmarks ON on input line 4772. +\c@Hy@tempcnt=\count106 +(/usr/local/texlive/2019/texmf-dist/tex/latex/url/url.sty +\Urlmuskip=\muskip11 +Package: url 2013/09/16 ver 3.4 Verb mode for urls, etc. +) +LaTeX Info: Redefining \url on input line 5125. +\XeTeXLinkMargin=\dimen114 +\Fld@menulength=\count107 +\Field@Width=\dimen115 +\Fld@charsize=\dimen116 +Package hyperref Info: Hyper figures OFF on input line 6380. +Package hyperref Info: Link nesting OFF on input line 6385. +Package hyperref Info: Hyper index ON on input line 6388. +Package hyperref Info: backreferencing OFF on input line 6395. +Package hyperref Info: Link coloring OFF on input line 6400. +Package hyperref Info: Link coloring with OCG OFF on input line 6405. +Package hyperref Info: PDF/A mode OFF on input line 6410. +LaTeX Info: Redefining \ref on input line 6450. +LaTeX Info: Redefining \pageref on input line 6454. +\Hy@abspage=\count108 +\c@Item=\count109 +\c@Hfootnote=\count110 +) +Package hyperref Info: Driver (autodetected): hpdftex. +(/usr/local/texlive/2019/texmf-dist/tex/latex/hyperref/hpdftex.def +File: hpdftex.def 2018/11/30 v6.88e Hyperref driver for pdfTeX +\Fld@listcount=\count111 +\c@bookmark@seq@number=\count112 +(/usr/local/texlive/2019/texmf-dist/tex/latex/oberdiek/rerunfilecheck.sty +Package: rerunfilecheck 2016/05/16 v1.8 Rerun checks for auxiliary files (HO) +Package uniquecounter Info: New unique counter `rerunfilecheck' on input line 2 +82. +) +\Hy@SectionHShift=\skip47 +) +Package hyperref Info: Option `breaklinks' set `true' on input line 35. +(/usr/local/texlive/2019/texmf-dist/tex/latex/geometry/geometry.sty +Package: geometry 2018/04/16 v5.8 Page Geometry +\Gm@cnth=\count113 +\Gm@cntv=\count114 +\c@Gm@tempcnt=\count115 +\Gm@bindingoffset=\dimen117 +\Gm@wd@mp=\dimen118 +\Gm@odd@mp=\dimen119 +\Gm@even@mp=\dimen120 +\Gm@layoutwidth=\dimen121 +\Gm@layoutheight=\dimen122 +\Gm@layouthoffset=\dimen123 +\Gm@layoutvoffset=\dimen124 +\Gm@dimlist=\toks24 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/graphics/color.sty +Package: color 2016/07/10 v1.1e Standard LaTeX Color (DPC) +(/usr/local/texlive/2019/texmf-dist/tex/latex/graphics-cfg/color.cfg +File: color.cfg 2016/01/02 v1.6 sample color configuration +) +Package color Info: Driver file: pdftex.def on input line 147. +(/usr/local/texlive/2019/texmf-dist/tex/latex/graphics-def/pdftex.def +File: pdftex.def 2018/01/08 v1.0l Graphics/color driver for pdftex +)) (/usr/local/texlive/2019/texmf-dist/tex/latex/fancyvrb/fancyvrb.sty +Package: fancyvrb 2019/01/15 +Style option: `fancyvrb' v3.2a <2019/01/15> (tvz) +\FV@CodeLineNo=\count116 +\FV@InFile=\read1 +\FV@TabBox=\box29 +\c@FancyVerbLine=\count117 +\FV@StepNumber=\count118 +\FV@OutFile=\write3 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/framed/framed.sty +Package: framed 2011/10/22 v 0.96: framed or shaded text with page breaks +\OuterFrameSep=\skip48 +\fb@frw=\dimen125 +\fb@frh=\dimen126 +\FrameRule=\dimen127 +\FrameSep=\dimen128 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/tools/longtable.sty +Package: longtable 2014/10/28 v4.11 Multi-page Table package (DPC)+ FMi change +\LTleft=\skip49 +\LTright=\skip50 +\LTpre=\skip51 +\LTpost=\skip52 +\LTchunksize=\count119 +\LTcapwidth=\dimen129 +\LT@head=\box30 +\LT@firsthead=\box31 +\LT@foot=\box32 +\LT@lastfoot=\box33 +\LT@cols=\count120 +\LT@rows=\count121 +\c@LT@tables=\count122 +\c@LT@chunks=\count123 +\LT@p@ftn=\toks25 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/booktabs/booktabs.sty +Package: booktabs 2016/04/27 v1.618033 publication quality tables +\heavyrulewidth=\dimen130 +\lightrulewidth=\dimen131 +\cmidrulewidth=\dimen132 +\belowrulesep=\dimen133 +\belowbottomsep=\dimen134 +\aboverulesep=\dimen135 +\abovetopsep=\dimen136 +\cmidrulesep=\dimen137 +\cmidrulekern=\dimen138 +\defaultaddspace=\dimen139 +\@cmidla=\count124 +\@cmidlb=\count125 +\@aboverulesep=\dimen140 +\@belowrulesep=\dimen141 +\@thisruleclass=\count126 +\@lastruleclass=\count127 +\@thisrulewidth=\dimen142 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/mdwtools/footnote.sty +Package: footnote 1997/01/28 1.13 Save footnotes around boxes +\fn@notes=\box34 +\fn@width=\dimen143 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/graphics/graphicx.sty +Package: graphicx 2017/06/01 v1.1a Enhanced LaTeX Graphics (DPC,SPQR) +(/usr/local/texlive/2019/texmf-dist/tex/latex/graphics/graphics.sty +Package: graphics 2017/06/25 v1.2c Standard LaTeX Graphics (DPC,SPQR) +(/usr/local/texlive/2019/texmf-dist/tex/latex/graphics/trig.sty +Package: trig 2016/01/03 v1.10 sin cos tan (DPC) +) (/usr/local/texlive/2019/texmf-dist/tex/latex/graphics-cfg/graphics.cfg +File: graphics.cfg 2016/06/04 v1.11 sample graphics configuration +) +Package graphics Info: Driver file: pdftex.def on input line 99. +) +\Gin@req@height=\dimen144 +\Gin@req@width=\dimen145 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/oberdiek/grffile.sty +Package: grffile 2017/06/30 v1.18 Extended file name support for graphics (HO) +Package grffile Info: Option `multidot' is set to `true'. +Package grffile Info: Option `extendedchars' is set to `false'. +Package grffile Info: Option `space' is set to `true'. +Package grffile Info: \Gin@ii of package `graphicx' fixed on input line 494. +) (/usr/local/texlive/2019/texmf-dist/tex/latex/tools/array.sty +Package: array 2018/12/30 v2.4k Tabular extension package (FMi) +\col@sep=\dimen146 +\ar@mcellbox=\box35 +\extrarowheight=\dimen147 +\NC@list=\toks26 +\extratabsurround=\skip53 +\backup@length=\skip54 +\ar@cellbox=\box36 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/multirow/multirow.sty +Package: multirow 2019/01/01 v2.4 Span multiple rows of a table +\multirow@colwidth=\skip55 +\multirow@cntb=\count128 +\multirow@dima=\skip56 +\bigstrutjot=\dimen148 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/wrapfig/wrapfig.sty +\wrapoverhang=\dimen149 +\WF@size=\dimen150 +\c@WF@wrappedlines=\count129 +\WF@box=\box37 +\WF@everypar=\toks27 +Package: wrapfig 2003/01/31 v 3.6 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/float/float.sty +Package: float 2001/11/08 v1.3d Float enhancements (AL) +\c@float@type=\count130 +\float@exts=\toks28 +\float@box=\box38 +\@float@everytoks=\toks29 +\@floatcapt=\box39 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/colortbl/colortbl.sty +Package: colortbl 2018/12/12 v1.0d Color table columns (DPC) +\everycr=\toks30 +\minrowclearance=\skip57 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/oberdiek/pdflscape.sty +Package: pdflscape 2016/05/14 v0.11 Display of landscape pages in PDF (HO) +(/usr/local/texlive/2019/texmf-dist/tex/latex/graphics/lscape.sty +Package: lscape 2000/10/22 v3.01 Landscape Pages (DPC) +) +Package pdflscape Info: Auto-detected driver: pdftex on input line 81. +) (/usr/local/texlive/2019/texmf-dist/tex/latex/tabu/tabu.sty +Package: tabu 2019/01/11 v2.9 - flexible LaTeX tabulars (FC+tabu-fixed) +(/usr/local/texlive/2019/texmf-dist/tex/latex/varwidth/varwidth.sty +Package: varwidth 2009/03/30 ver 0.92; Variable-width minipages +\@vwid@box=\box40 +\sift@deathcycles=\count131 +\@vwid@loff=\dimen151 +\@vwid@roff=\dimen152 +) +\c@taburow=\count132 +\tabu@nbcols=\count133 +\tabu@cnt=\count134 +\tabu@Xcol=\count135 +\tabu@alloc=\count136 +\tabu@nested=\count137 +\tabu@target=\dimen153 +\tabu@spreadtarget=\dimen154 +\tabu@naturalX=\dimen155 +\tabucolX=\dimen156 +\tabu@Xsum=\dimen157 +\extrarowdepth=\dimen158 +\abovetabulinesep=\dimen159 +\belowtabulinesep=\dimen160 +\tabustrutrule=\dimen161 +\tabu@thebody=\toks31 +\tabu@footnotes=\toks32 +\tabu@box=\box41 +\tabu@arstrutbox=\box42 +\tabu@hleads=\box43 +\tabu@vleads=\box44 +\tabu@cellskip=\skip58 +) +(/usr/local/texlive/2019/texmf-dist/tex/latex/threeparttable/threeparttable.sty +Package: threeparttable 2003/06/13 v 3.0 +\@tempboxb=\box45 +) +(/usr/local/texlive/2019/texmf-dist/tex/latex/threeparttablex/threeparttablex.s +ty +Package: threeparttablex 2013/07/23 v0.3 by daleif +(/usr/local/texlive/2019/texmf-dist/tex/latex/environ/environ.sty +Package: environ 2014/05/04 v0.3 A new way to define environments +(/usr/local/texlive/2019/texmf-dist/tex/latex/trimspaces/trimspaces.sty +Package: trimspaces 2009/09/17 v1.1 Trim spaces around a token list +)) +\TPTL@width=\skip59 +) (/usr/local/texlive/2019/texmf-dist/tex/generic/ulem/ulem.sty +\UL@box=\box46 +\UL@hyphenbox=\box47 +\UL@skip=\skip60 +\UL@hook=\toks33 +\UL@height=\dimen162 +\UL@pe=\count138 +\UL@pixel=\dimen163 +\ULC@box=\box48 +Package: ulem 2012/05/18 +\ULdepth=\dimen164 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/makecell/makecell.sty +Package: makecell 2009/08/03 V0.1e Managing of Tab Column Heads and Cells +\rotheadsize=\dimen165 +\c@nlinenum=\count139 +\TeXr@lab=\toks34 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/xcolor/xcolor.sty +Package: xcolor 2016/05/11 v2.12 LaTeX color extensions (UK) +(/usr/local/texlive/2019/texmf-dist/tex/latex/graphics-cfg/color.cfg +File: color.cfg 2016/01/02 v1.6 sample color configuration +) +Package xcolor Info: Driver file: pdftex.def on input line 225. +LaTeX Info: Redefining \color on input line 709. +Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1348. +Package xcolor Info: Model `hsb' substituted by `rgb' on input line 1352. +Package xcolor Info: Model `RGB' extended on input line 1364. +Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1366. +Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1367. +Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1368. +Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1369. +Package xcolor Info: Model `Gray' substituted by `gray' on input line 1370. +Package xcolor Info: Model `wave' substituted by `hsb' on input line 1371. +) (./cleaningFeederData.aux) +\openout1 = `cleaningFeederData.aux'. + +LaTeX Font Info: Checking defaults for OML/cmm/m/it on input line 136. +LaTeX Font Info: ... okay on input line 136. +LaTeX Font Info: Checking defaults for T1/cmr/m/n on input line 136. +LaTeX Font Info: ... okay on input line 136. +LaTeX Font Info: Checking defaults for OT1/cmr/m/n on input line 136. +LaTeX Font Info: ... okay on input line 136. +LaTeX Font Info: Checking defaults for OMS/cmsy/m/n on input line 136. +LaTeX Font Info: ... okay on input line 136. +LaTeX Font Info: Checking defaults for OMX/cmex/m/n on input line 136. +LaTeX Font Info: ... okay on input line 136. +LaTeX Font Info: Checking defaults for U/cmr/m/n on input line 136. +LaTeX Font Info: ... okay on input line 136. +LaTeX Font Info: Checking defaults for TS1/cmr/m/n on input line 136. +LaTeX Font Info: Try loading font information for TS1+cmr on input line 136. + +(/usr/local/texlive/2019/texmf-dist/tex/latex/base/ts1cmr.fd +File: ts1cmr.fd 2014/09/29 v2.5h Standard LaTeX font definitions +) +LaTeX Font Info: ... okay on input line 136. +LaTeX Font Info: Checking defaults for PD1/pdf/m/n on input line 136. +LaTeX Font Info: ... okay on input line 136. +LaTeX Font Info: Checking defaults for PU/pdf/m/n on input line 136. +LaTeX Font Info: ... okay on input line 136. +LaTeX Font Info: Try loading font information for T1+lmr on input line 136. +(/usr/local/texlive/2019/texmf-dist/tex/latex/lm/t1lmr.fd +File: t1lmr.fd 2009/10/30 v1.6 Font defs for Latin Modern +) +LaTeX Info: Redefining \microtypecontext on input line 136. +Package microtype Info: Generating PDF output. +Package microtype Info: Character protrusion enabled (level 2). +Package microtype Info: Using protrusion set `basicmath'. +Package microtype Info: Automatic font expansion enabled (level 2), +(microtype) stretch: 20, shrink: 20, step: 1, non-selected. +Package microtype Info: Using default expansion set `basictext'. +Package microtype Info: No adjustment of tracking. +Package microtype Info: No adjustment of interword spacing. +Package microtype Info: No adjustment of character kerning. +(/usr/local/texlive/2019/texmf-dist/tex/latex/microtype/mt-cmr.cfg +File: mt-cmr.cfg 2013/05/19 v2.2 microtype config. file: Computer Modern Roman +(RS) +) +\AtBeginShipoutBox=\box49 +Package hyperref Info: Link coloring OFF on input line 136. +(/usr/local/texlive/2019/texmf-dist/tex/latex/hyperref/nameref.sty +Package: nameref 2016/05/21 v2.44 Cross-referencing by name of section +(/usr/local/texlive/2019/texmf-dist/tex/generic/oberdiek/gettitlestring.sty +Package: gettitlestring 2016/05/16 v1.5 Cleanup title references (HO) +) +\c@section@level=\count140 +) +LaTeX Info: Redefining \ref on input line 136. +LaTeX Info: Redefining \pageref on input line 136. +LaTeX Info: Redefining \nameref on input line 136. +(./cleaningFeederData.out) (./cleaningFeederData.out) +\@outlinefile=\write4 +\openout4 = `cleaningFeederData.out'. + +*geometry* driver: auto-detecting +*geometry* detected driver: pdftex +*geometry* verbose mode - [ preamble ] result: +* driver: pdftex +* paper: <default> +* layout: <same size as paper> +* layoutoffset:(h,v)=(0.0pt,0.0pt) +* modes: +* h-part:(L,W,R)=(72.26999pt, 469.75502pt, 72.26999pt) +* v-part:(T,H,B)=(72.26999pt, 650.43001pt, 72.26999pt) +* \paperwidth=614.295pt +* \paperheight=794.96999pt +* \textwidth=469.75502pt +* \textheight=650.43001pt +* \oddsidemargin=0.0pt +* \evensidemargin=0.0pt +* \topmargin=-37.0pt +* \headheight=12.0pt +* \headsep=25.0pt +* \topskip=10.0pt +* \footskip=30.0pt +* \marginparwidth=65.0pt +* \marginparsep=11.0pt +* \columnsep=10.0pt +* \skip\footins=9.0pt plus 4.0pt minus 2.0pt +* \hoffset=0.0pt +* \voffset=0.0pt +* \mag=1000 +* \@twocolumnfalse +* \@twosidefalse +* \@mparswitchfalse +* \@reversemarginfalse +* (1in=72.27pt=25.4mm, 1cm=28.453pt) + +(/usr/local/texlive/2019/texmf-dist/tex/context/base/mkii/supp-pdf.mkii +[Loading MPS to PDF converter (version 2006.09.02).] +\scratchcounter=\count141 +\scratchdimen=\dimen166 +\scratchbox=\box50 +\nofMPsegments=\count142 +\nofMParguments=\count143 +\everyMPshowfont=\toks35 +\MPscratchCnt=\count144 +\MPscratchDim=\dimen167 +\MPnumerator=\count145 +\makeMPintoPDFobject=\count146 +\everyMPtoPDFconversion=\toks36 +) (/usr/local/texlive/2019/texmf-dist/tex/latex/oberdiek/epstopdf-base.sty +Package: epstopdf-base 2016/05/15 v2.6 Base part for package epstopdf +(/usr/local/texlive/2019/texmf-dist/tex/latex/oberdiek/grfext.sty +Package: grfext 2016/05/16 v1.2 Manage graphics extensions (HO) +) +Package epstopdf-base Info: Redefining graphics rule for `.eps' on input line 4 +38. +Package grfext Info: Graphics extension search list: +(grfext) [.pdf,.png,.jpg,.mps,.jpeg,.jbig2,.jb2,.PDF,.PNG,.JPG,.JPE +G,.JBIG2,.JB2,.eps] +(grfext) \AppendGraphicsExtensions on input line 456. +(/usr/local/texlive/2019/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg +File: epstopdf-sys.cfg 2010/07/13 v1.3 Configuration of (r)epstopdf for TeX Liv +e +)) +LaTeX Font Info: Try loading font information for OT1+lmr on input line 138. + +(/usr/local/texlive/2019/texmf-dist/tex/latex/lm/ot1lmr.fd +File: ot1lmr.fd 2009/10/30 v1.6 Font defs for Latin Modern +) +LaTeX Font Info: Try loading font information for OML+lmm on input line 138. + +(/usr/local/texlive/2019/texmf-dist/tex/latex/lm/omllmm.fd +File: omllmm.fd 2009/10/30 v1.6 Font defs for Latin Modern +) +LaTeX Font Info: Try loading font information for OMS+lmsy on input line 138 +. +(/usr/local/texlive/2019/texmf-dist/tex/latex/lm/omslmsy.fd +File: omslmsy.fd 2009/10/30 v1.6 Font defs for Latin Modern +) +LaTeX Font Info: Try loading font information for OMX+lmex on input line 138 +. +(/usr/local/texlive/2019/texmf-dist/tex/latex/lm/omxlmex.fd +File: omxlmex.fd 2009/10/30 v1.6 Font defs for Latin Modern +) +LaTeX Font Info: External font `lmex10' loaded for size +(Font) <12> on input line 138. +LaTeX Font Info: External font `lmex10' loaded for size +(Font) <8> on input line 138. +LaTeX Font Info: External font `lmex10' loaded for size +(Font) <6> on input line 138. +LaTeX Font Info: Try loading font information for U+msa on input line 138. +(/usr/local/texlive/2019/texmf-dist/tex/latex/amsfonts/umsa.fd +File: umsa.fd 2013/01/14 v3.01 AMS symbols A +) (/usr/local/texlive/2019/texmf-dist/tex/latex/microtype/mt-msa.cfg +File: mt-msa.cfg 2006/02/04 v1.1 microtype config. file: AMS symbols (a) (RS) +) +LaTeX Font Info: Try loading font information for U+msb on input line 138. +(/usr/local/texlive/2019/texmf-dist/tex/latex/amsfonts/umsb.fd +File: umsb.fd 2013/01/14 v3.01 AMS symbols B +) (/usr/local/texlive/2019/texmf-dist/tex/latex/microtype/mt-msb.cfg +File: mt-msb.cfg 2005/06/01 v1.0 microtype config. file: AMS symbols (b) (RS) +) +Overfull \hbox (79.61282pt too wide) in paragraph at lines 138--138 + [][] + [] + +(./cleaningFeederData.toc +LaTeX Font Info: External font `lmex10' loaded for size +(Font) <10> on input line 3. +LaTeX Font Info: External font `lmex10' loaded for size +(Font) <7> on input line 3. +LaTeX Font Info: External font `lmex10' loaded for size +(Font) <5> on input line 3. +) +\tf@toc=\write5 +\openout5 = `cleaningFeederData.toc'. + + +Package xcolor Warning: Incompatible color definition on input line 156. + +LaTeX Font Info: Try loading font information for T1+lmtt on input line 157. + +(/usr/local/texlive/2019/texmf-dist/tex/latex/lm/t1lmtt.fd +File: t1lmtt.fd 2009/10/30 v1.6 Font defs for Latin Modern +) +LaTeX Font Info: Font shape `T1/lmtt/bx/n' in size <10> not available +(Font) Font shape `T1/lmtt/b/n' tried instead on input line 158. + +Package xcolor Warning: Incompatible color definition on input line 162. + + +Package xcolor Warning: Incompatible color definition on input line 162. + +[1 + +{/usr/local/texlive/2019/texmf-var/fonts/map/pdftex/updmap/pdftex.map}] + +Package xcolor Warning: Incompatible color definition on input line 183. + +LaTeX Font Info: Try loading font information for TS1+lmtt on input line 194 +. +(/usr/local/texlive/2019/texmf-dist/tex/latex/lm/ts1lmtt.fd +File: ts1lmtt.fd 2009/10/30 v1.6 Font defs for Latin Modern +) + +Package xcolor Warning: Incompatible color definition on input line 200. + + +Package xcolor Warning: Incompatible color definition on input line 200. + + +Overfull \hbox (42.00665pt too wide) in paragraph at lines 206--225 + [][] + [] + + +Package xcolor Warning: Incompatible color definition on input line 229. + + +Package xcolor Warning: Incompatible color definition on input line 240. + + +Package xcolor Warning: Incompatible color definition on input line 240. + + +Package xcolor Warning: Incompatible color definition on input line 285. + +[2pdfTeX warning (ext4): destination with the same identifier (name{table.1}) h +as been already used, duplicate ignored + +\AtBegShi@Output ...ipout \box \AtBeginShipoutBox + \fi \fi +l.285 \begin{Shaded} + ] + +Package xcolor Warning: Incompatible color definition on input line 291. + + +Package xcolor Warning: Incompatible color definition on input line 291. + +<cleaningFeederData_files/figure-latex/histo-1.pdf, id=109, 459.7175pt x 313.17 +pt> +File: cleaningFeederData_files/figure-latex/histo-1.pdf Graphic file (type pdf) + +<use cleaningFeederData_files/figure-latex/histo-1.pdf> +Package pdftex.def Info: cleaningFeederData_files/figure-latex/histo-1.pdf use +d on input line 293. +(pdftex.def) Requested size: 459.7304pt x 313.17877pt. + +Package xcolor Warning: Incompatible color definition on input line 297. + +[3pdfTeX warning (ext4): destination with the same identifier (name{table.2}) h +as been already used, duplicate ignored + +\AtBegShi@Output ...ipout \box \AtBeginShipoutBox + \fi \fi +l.297 \begin{Shaded} + <./cleaningFeederData_files/figure-latex/histo-1.pdf>] + +Package xcolor Warning: Incompatible color definition on input line 304. + + +Package xcolor Warning: Incompatible color definition on input line 304. + +<cleaningFeederData_files/figure-latex/kwProfiles-1.pdf, id=122, 453.695pt x 31 +3.17pt> +File: cleaningFeederData_files/figure-latex/kwProfiles-1.pdf Graphic file (type + pdf) +<use cleaningFeederData_files/figure-latex/kwProfiles-1.pdf> +Package pdftex.def Info: cleaningFeederData_files/figure-latex/kwProfiles-1.pdf + used on input line 306. +(pdftex.def) Requested size: 453.6939pt x 313.16922pt. + +Package xcolor Warning: Incompatible color definition on input line 315. + + +Package xcolor Warning: Incompatible color definition on input line 323. + + +Package xcolor Warning: Incompatible color definition on input line 323. + +<cleaningFeederData_files/figure-latex/missingVis-1.pdf, id=123, 453.695pt x 31 +6.18124pt> +File: cleaningFeederData_files/figure-latex/missingVis-1.pdf Graphic file (type + pdf) +<use cleaningFeederData_files/figure-latex/missingVis-1.pdf> +Package pdftex.def Info: cleaningFeederData_files/figure-latex/missingVis-1.pdf + used on input line 325. +(pdftex.def) Requested size: 453.6939pt x 316.18047pt. +[4 <./cleaningFeederData_files/figure-latex/kwProfiles-1.pdf>] + +Package xcolor Warning: Incompatible color definition on input line 331. + + +Package xcolor Warning: Incompatible color definition on input line 342. + + +Package xcolor Warning: Incompatible color definition on input line 342. + +<cleaningFeederData_files/figure-latex/aggVis-1.pdf, id=136, 453.695pt x 316.18 +124pt> +File: cleaningFeederData_files/figure-latex/aggVis-1.pdf Graphic file (type pdf +) +<use cleaningFeederData_files/figure-latex/aggVis-1.pdf> +Package pdftex.def Info: cleaningFeederData_files/figure-latex/aggVis-1.pdf us +ed on input line 344. +(pdftex.def) Requested size: 453.6939pt x 316.18047pt. + +Package xcolor Warning: Incompatible color definition on input line 346. + +[5 <./cleaningFeederData_files/figure-latex/missingVis-1.pdf>] + +Package xcolor Warning: Incompatible color definition on input line 354. + + +Package xcolor Warning: Incompatible color definition on input line 354. + +<cleaningFeederData_files/figure-latex/aggVis-2.pdf, id=149, 453.695pt x 316.18 +124pt> +File: cleaningFeederData_files/figure-latex/aggVis-2.pdf Graphic file (type pdf +) +<use cleaningFeederData_files/figure-latex/aggVis-2.pdf> +Package pdftex.def Info: cleaningFeederData_files/figure-latex/aggVis-2.pdf us +ed on input line 356. +(pdftex.def) Requested size: 453.6939pt x 316.18047pt. + +Package xcolor Warning: Incompatible color definition on input line 358. + +[6 <./cleaningFeederData_files/figure-latex/aggVis-1.pdf>] + +Package xcolor Warning: Incompatible color definition on input line 365. + + +Package xcolor Warning: Incompatible color definition on input line 365. + +<cleaningFeederData_files/figure-latex/aggVis-3.pdf, id=163, 459.7175pt x 316.1 +8124pt> +File: cleaningFeederData_files/figure-latex/aggVis-3.pdf Graphic file (type pdf +) +<use cleaningFeederData_files/figure-latex/aggVis-3.pdf> +Package pdftex.def Info: cleaningFeederData_files/figure-latex/aggVis-3.pdf us +ed on input line 367. +(pdftex.def) Requested size: 459.74442pt x 316.19975pt. +[7 <./cleaningFeederData_files/figure-latex/aggVis-2.pdf>] + +Package xcolor Warning: Incompatible color definition on input line 374. + + +Package xcolor Warning: Incompatible color definition on input line 382. + + +Package xcolor Warning: Incompatible color definition on input line 382. + + +Package xcolor Warning: Incompatible color definition on input line 401. + +[8 <./cleaningFeederData_files/figure-latex/aggVis-3.pdf>] + +Package xcolor Warning: Incompatible color definition on input line 408. + + +Package xcolor Warning: Incompatible color definition on input line 408. + +<cleaningFeederData_files/figure-latex/testDateTimes-1.pdf, id=189, 453.695pt x + 316.18124pt> +File: cleaningFeederData_files/figure-latex/testDateTimes-1.pdf Graphic file (t +ype pdf) +<use cleaningFeederData_files/figure-latex/testDateTimes-1.pdf> +Package pdftex.def Info: cleaningFeederData_files/figure-latex/testDateTimes-1. +pdf used on input line 410. +(pdftex.def) Requested size: 453.6939pt x 316.18047pt. + +Package xcolor Warning: Incompatible color definition on input line 416. + + +Package xcolor Warning: Incompatible color definition on input line 424. + + +Package xcolor Warning: Incompatible color definition on input line 424. + +<cleaningFeederData_files/figure-latex/missingProfiles-1.pdf, id=190, 457.71pt +x 316.18124pt> +File: cleaningFeederData_files/figure-latex/missingProfiles-1.pdf Graphic file +(type pdf) +<use cleaningFeederData_files/figure-latex/missingProfiles-1.pdf> +Package pdftex.def Info: cleaningFeederData_files/figure-latex/missingProfiles- +1.pdf used on input line 426. +(pdftex.def) Requested size: 457.72984pt x 316.19493pt. +[9 <./cleaningFeederData_files/figure-latex/testDateTimes-1.pdf>] +LaTeX Font Info: Try loading font information for TS1+lmr on input line 435. + +(/usr/local/texlive/2019/texmf-dist/tex/latex/lm/ts1lmr.fd +File: ts1lmr.fd 2009/10/30 v1.6 Font defs for Latin Modern +) [10 <./cleaningFeederData_files/figure-latex/missingProfiles-1.pdf>] + +Package xcolor Warning: Incompatible color definition on input line 532. + + +Package xcolor Warning: Incompatible color definition on input line 540. + + +Package xcolor Warning: Incompatible color definition on input line 540. + + +Package xcolor Warning: Incompatible color definition on input line 545. + +[11] + +Package xcolor Warning: Incompatible color definition on input line 551. + + +Package xcolor Warning: Incompatible color definition on input line 551. + + +Package xcolor Warning: Incompatible color definition on input line 555. + + +Package xcolor Warning: Incompatible color definition on input line 602. + + +Package xcolor Warning: Incompatible color definition on input line 602. + + +Package xcolor Warning: Incompatible color definition on input line 606. + +[12] + +Package xcolor Warning: Incompatible color definition on input line 610. + + +Package xcolor Warning: Incompatible color definition on input line 610. + + +Package xcolor Warning: Incompatible color definition on input line 612. + + +Package xcolor Warning: Incompatible color definition on input line 625. + + +Package xcolor Warning: Incompatible color definition on input line 625. + + +Package xcolor Warning: Incompatible color definition on input line 627. + + +Package xcolor Warning: Incompatible color definition on input line 634. + + +Package xcolor Warning: Incompatible color definition on input line 634. + + +Package xcolor Warning: Incompatible color definition on input line 639. + + +Package xcolor Warning: Incompatible color definition on input line 643. + + +Package xcolor Warning: Incompatible color definition on input line 643. + + +Package xcolor Warning: Incompatible color definition on input line 645. + + +Package xcolor Warning: Incompatible color definition on input line 659. + + +Package xcolor Warning: Incompatible color definition on input line 659. + + +Package xcolor Warning: Incompatible color definition on input line 664. + + +Package xcolor Warning: Incompatible color definition on input line 681. + + +Package xcolor Warning: Incompatible color definition on input line 681. + + +Package xcolor Warning: Incompatible color definition on input line 686. + +[13] + +Package xcolor Warning: Incompatible color definition on input line 736. + + +Package xcolor Warning: Incompatible color definition on input line 736. + + +Package xcolor Warning: Incompatible color definition on input line 740. + + +Package xcolor Warning: Incompatible color definition on input line 744. + + +Package xcolor Warning: Incompatible color definition on input line 744. + +[14] +Package atveryend Info: Empty hook `BeforeClearDocument' on input line 778. +[15] +Package atveryend Info: Empty hook `AfterLastShipout' on input line 778. +(./cleaningFeederData.aux) +Package atveryend Info: Executing hook `AtVeryEndDocument' on input line 778. +Package atveryend Info: Executing hook `AtEndAfterFileList' on input line 778. +Package rerunfilecheck Info: File `cleaningFeederData.out' has not changed. +(rerunfilecheck) Checksum: 9E0383F59F906E2FFD876F671F1E7F20;2058. + ) +Here is how much of TeX's memory you used: + 14920 strings out of 492616 + 218218 string characters out of 6129481 + 348476 words of memory out of 5000000 + 18466 multiletter control sequences out of 15000+600000 + 58284 words of font info for 85 fonts, out of 8000000 for 9000 + 1141 hyphenation exceptions out of 8191 + 35i,9n,38p,826b,456s stack positions out of 5000i,500n,10000p,200000b,80000s +{/usr/local/texlive/2019/texmf-dist/fonts/enc/dvips/lm/lm-ts1.enc}{/usr/local +/texlive/2019/texmf-dist/fonts/enc/dvips/lm/lm-ec.enc}</usr/local/texlive/2019/ +texmf-dist/fonts/type1/public/lm/lmbx10.pfb></usr/local/texlive/2019/texmf-dist +/fonts/type1/public/lm/lmbx12.pfb></usr/local/texlive/2019/texmf-dist/fonts/typ +e1/public/lm/lmr10.pfb></usr/local/texlive/2019/texmf-dist/fonts/type1/public/l +m/lmr12.pfb></usr/local/texlive/2019/texmf-dist/fonts/type1/public/lm/lmr17.pfb +></usr/local/texlive/2019/texmf-dist/fonts/type1/public/lm/lmri10.pfb></usr/loc +al/texlive/2019/texmf-dist/fonts/type1/public/lm/lmtk10.pfb></usr/local/texlive +/2019/texmf-dist/fonts/type1/public/lm/lmtt10.pfb></usr/local/texlive/2019/texm +f-dist/fonts/type1/public/lm/lmtti10.pfb> +Output written on cleaningFeederData.pdf (15 pages, 792482 bytes). +PDF statistics: + 327 PDF objects out of 1000 (max. 8388607) + 270 compressed objects within 3 object streams + 62 named destinations out of 1000 (max. 500000) + 12457 words of extra memory for PDF output out of 14400 (max. 10000000) + diff --git a/isleOfWight/cleaningFeederData.pdf b/isleOfWight/cleaningFeederData.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2505bc49de670a121c78705a5890d02f13a5dfce Binary files /dev/null and b/isleOfWight/cleaningFeederData.pdf differ diff --git a/isleOfWight/cleaningFeederData.tex b/isleOfWight/cleaningFeederData.tex new file mode 100644 index 0000000000000000000000000000000000000000..b8b6e5bd70a22d7ec32a078dfc463d6c27531859 --- /dev/null +++ b/isleOfWight/cleaningFeederData.tex @@ -0,0 +1,778 @@ +\PassOptionsToPackage{unicode=true}{hyperref} % options for packages loaded elsewhere +\PassOptionsToPackage{hyphens}{url} +% +\documentclass[]{article} +\usepackage{lmodern} +\usepackage{amssymb,amsmath} +\usepackage{ifxetex,ifluatex} +\usepackage{fixltx2e} % provides \textsubscript +\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex + \usepackage[T1]{fontenc} + \usepackage[utf8]{inputenc} + \usepackage{textcomp} % provides euro and other symbols +\else % if luatex or xelatex + \usepackage{unicode-math} + \defaultfontfeatures{Ligatures=TeX,Scale=MatchLowercase} +\fi +% use upquote if available, for straight quotes in verbatim environments +\IfFileExists{upquote.sty}{\usepackage{upquote}}{} +% use microtype if available +\IfFileExists{microtype.sty}{% +\usepackage[]{microtype} +\UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts +}{} +\IfFileExists{parskip.sty}{% +\usepackage{parskip} +}{% else +\setlength{\parindent}{0pt} +\setlength{\parskip}{6pt plus 2pt minus 1pt} +} +\usepackage{hyperref} +\hypersetup{ + pdftitle={Cleaning Feeder Data}, + pdfauthor={Ben Anderson (b.anderson@soton.ac.uk), SERG, Energy \& Climate Change, University of Southampton}, + pdfborder={0 0 0}, + breaklinks=true} +\urlstyle{same} % don't use monospace font for urls +\usepackage[margin=1in]{geometry} +\usepackage{color} +\usepackage{fancyvrb} +\newcommand{\VerbBar}{|} +\newcommand{\VERB}{\Verb[commandchars=\\\{\}]} +\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}} +% Add ',fontsize=\small' for more characters per line +\usepackage{framed} +\definecolor{shadecolor}{RGB}{248,248,248} +\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}} +\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}} +\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}} +\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}} +\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}} +\newcommand{\BuiltInTok}[1]{#1} +\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}} +\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}} +\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}} +\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}} +\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}} +\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}} +\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}} +\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}} +\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}} +\newcommand{\ExtensionTok}[1]{#1} +\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}} +\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}} +\newcommand{\ImportTok}[1]{#1} +\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}} +\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}} +\newcommand{\NormalTok}[1]{#1} +\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}} +\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}} +\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}} +\newcommand{\RegionMarkerTok}[1]{#1} +\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}} +\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}} +\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}} +\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}} +\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}} +\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}} +\usepackage{longtable,booktabs} +% Fix footnotes in tables (requires footnote package) +\IfFileExists{footnote.sty}{\usepackage{footnote}\makesavenoteenv{longtable}}{} +\usepackage{graphicx,grffile} +\makeatletter +\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi} +\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi} +\makeatother +% Scale images if necessary, so that they will not overflow the page +% margins by default, and it is still possible to overwrite the defaults +% using explicit options in \includegraphics[width, height, ...]{} +\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio} +\setlength{\emergencystretch}{3em} % prevent overfull lines +\providecommand{\tightlist}{% + \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}} +\setcounter{secnumdepth}{5} +% Redefines (sub)paragraphs to behave more like sections +\ifx\paragraph\undefined\else +\let\oldparagraph\paragraph +\renewcommand{\paragraph}[1]{\oldparagraph{#1}\mbox{}} +\fi +\ifx\subparagraph\undefined\else +\let\oldsubparagraph\subparagraph +\renewcommand{\subparagraph}[1]{\oldsubparagraph{#1}\mbox{}} +\fi + +% set default figure placement to htbp +\makeatletter +\def\fps@figure{htbp} +\makeatother + +\usepackage{etoolbox} +\makeatletter +\providecommand{\subtitle}[1]{% add subtitle to \maketitle + \apptocmd{\@title}{\par {\large #1 \par}}{}{} +} +\makeatother +\usepackage{booktabs} +\usepackage{longtable} +\usepackage{array} +\usepackage{multirow} +\usepackage{wrapfig} +\usepackage{float} +\usepackage{colortbl} +\usepackage{pdflscape} +\usepackage{tabu} +\usepackage{threeparttable} +\usepackage{threeparttablex} +\usepackage[normalem]{ulem} +\usepackage{makecell} +\usepackage{xcolor} + +\title{Cleaning Feeder Data} +\providecommand{\subtitle}[1]{} +\subtitle{Code and notes} +\author{Ben Anderson (\href{mailto:b.anderson@soton.ac.uk}{\nolinkurl{b.anderson@soton.ac.uk}}), \href{http://www.energy.soton.ac.uk/}{SERG, Energy \& Climate Change}, University of Southampton} +\date{Last run at: 2020-07-07 17:25:30} + +\begin{document} +\maketitle + +{ +\setcounter{tocdepth}{2} +\tableofcontents +} +\hypertarget{intro}{% +\section{Intro}\label{intro}} + +We have some feeder data. There seem to be NAs and missing time stamps. We want to select the `best' (i.e most complete) days within a day-of-the-week/season/year sampling frame. + +\hypertarget{data-prep}{% +\section{Data prep}\label{data-prep}} + +\hypertarget{load-data}{% +\subsection{Load data}\label{load-data}} + +Loading from \textasciitilde{}/Dropbox/Ben\_IOW\_SS.csv\ldots{} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{dt<-}\StringTok{ }\NormalTok{data.table}\OperatorTok{::}\KeywordTok{fread}\NormalTok{(dFile)} +\NormalTok{dt[, V1 }\OperatorTok{:}\ErrorTok{=}\StringTok{ }\OtherTok{NULL}\NormalTok{] }\CommentTok{#drop} +\KeywordTok{head}\NormalTok{(dt)} +\end{Highlighting} +\end{Shaded} + +\begin{verbatim} +## DateTime Value region sub_region Date Time Year Month +## 1: 2017-01-01T00:30:00Z 99 BINS C1T0 2017-01-01 00:30:00 2017 1 +## 2: 2017-01-01T01:00:00Z 91 BINS C1T0 2017-01-01 01:00:00 2017 1 +## 3: 2017-01-01T01:45:00Z 84 BINS C1T0 2017-01-01 01:45:00 2017 1 +## 4: 2017-01-01T02:15:00Z 77 BINS C1T0 2017-01-01 02:15:00 2017 1 +## 5: 2017-01-01T02:45:00Z 69 BINS C1T0 2017-01-01 02:45:00 2017 1 +## 6: 2017-01-01T05:45:00Z 77 BINS C1T0 2017-01-01 05:45:00 2017 1 +## Day Hour +## 1: 1 0 +## 2: 1 1 +## 3: 1 1 +## 4: 1 2 +## 5: 1 2 +## 6: 1 5 +\end{verbatim} + +Do some data prep + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{dt[, rDateTime }\OperatorTok{:}\ErrorTok{=}\StringTok{ }\NormalTok{lubridate}\OperatorTok{::}\KeywordTok{as_datetime}\NormalTok{(DateTime)]} +\NormalTok{dt[, rTime }\OperatorTok{:}\ErrorTok{=}\StringTok{ }\NormalTok{hms}\OperatorTok{::}\KeywordTok{as_hms}\NormalTok{(rDateTime)]} +\NormalTok{dt[, rDate }\OperatorTok{:}\ErrorTok{=}\StringTok{ }\KeywordTok{as.Date}\NormalTok{(rDateTime)]} +\NormalTok{dt[, rYear }\OperatorTok{:}\ErrorTok{=}\StringTok{ }\NormalTok{lubridate}\OperatorTok{::}\KeywordTok{year}\NormalTok{(rDate)]} +\NormalTok{dt[, rDoW }\OperatorTok{:}\ErrorTok{=}\StringTok{ }\NormalTok{lubridate}\OperatorTok{::}\KeywordTok{wday}\NormalTok{(rDate, }\DataTypeTok{label =} \OtherTok{TRUE}\NormalTok{)]} +\NormalTok{dt[, kW }\OperatorTok{:}\ErrorTok{=}\StringTok{ }\NormalTok{Value] }\CommentTok{# for clarity} +\NormalTok{dt[, Value }\OperatorTok{:}\ErrorTok{=}\StringTok{ }\OtherTok{NULL}\NormalTok{] }\CommentTok{#drop} + +\NormalTok{dt <-}\StringTok{ }\KeywordTok{addSeason}\NormalTok{(dt, }\DataTypeTok{dateVar =} \StringTok{"rDate"}\NormalTok{, }\DataTypeTok{h =} \StringTok{"N"}\NormalTok{)} +\CommentTok{#skimr::skim(dt) doesn't play nicely with latex} +\NormalTok{st <-}\StringTok{ }\KeywordTok{summary}\NormalTok{(dt[,.(rYear, rDateTime, sub_region, kW, rDoW, season)])} +\NormalTok{kableExtra}\OperatorTok{::}\KeywordTok{kable}\NormalTok{(st,} + \DataTypeTok{caption =} \StringTok{"Summary of loaded and processed data"}\NormalTok{) }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{kable_styling}\NormalTok{()} +\end{Highlighting} +\end{Shaded} + +\begin{table} + +\caption{\label{tab:dataSummary}Summary of loaded and processed data} +\centering +\begin{tabular}[t]{l|l|l|l|l|l|l} +\hline + & rYear & rDateTime & sub\_region & kW & rDoW & season\\ +\hline + & Min. :2017 & Min. :2017-01-01 00:00:00 & Length:36280 & Min. : 0.00 & Sun:5304 & Spring:18404\\ +\hline + & 1st Qu.:2017 & 1st Qu.:2017-02-13 06:30:00 & Class :character & 1st Qu.: 53.50 & Mon:5082 & Summer: 5451\\ +\hline + & Median :2017 & Median :2017-03-27 12:22:30 & Mode :character & Median : 73.67 & Tue:5301 & Autumn: 0\\ +\hline + & Mean :2017 & Mean :2017-03-29 12:58:45 & NA & Mean : 76.09 & Wed:5131 & Winter:12425\\ +\hline + & 3rd Qu.:2017 & 3rd Qu.:2017-05-12 00:41:15 & NA & 3rd Qu.: 96.00 & Thu:5190 & NA\\ +\hline + & Max. :2017 & Max. :2017-06-30 23:45:00 & NA & Max. :241.25 & Fri:5328 & NA\\ +\hline + & NA & NA & NA & NA & Sat:4944 & NA\\ +\hline +\end{tabular} +\end{table} + +Do a sense check by season \& feeder. + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{t <-}\StringTok{ }\NormalTok{dt[, .(}\DataTypeTok{mean_kW =} \KeywordTok{mean}\NormalTok{(kW),} + \DataTypeTok{nObs =}\NormalTok{ .N), keyby =}\StringTok{ }\NormalTok{.(rYear, sub_region,season)]} + +\NormalTok{NAt <-}\StringTok{ }\NormalTok{dt[}\KeywordTok{is.na}\NormalTok{(kW), .(}\DataTypeTok{nObs =}\NormalTok{ .N), keyby =}\StringTok{ }\NormalTok{.(rYear, sub_region,season)]} + +\NormalTok{kableExtra}\OperatorTok{::}\KeywordTok{kable}\NormalTok{(t,} + \DataTypeTok{caption =} \StringTok{"Sense check on raw data"}\NormalTok{) }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{kable_styling}\NormalTok{()} +\end{Highlighting} +\end{Shaded} + +\begin{table} + +\caption{\label{tab:senseCheck}Sense check on raw data} +\centering +\begin{tabular}[t]{r|l|l|r|r} +\hline +rYear & sub\_region & season & mean\_kW & nObs\\ +\hline +2017 & C1T0 & Spring & 70.263621 & 3168\\ +\hline +2017 & C1T0 & Summer & 55.561198 & 939\\ +\hline +2017 & C1T0 & Winter & 99.396705 & 2090\\ +\hline +2017 & C2T0 & Spring & 77.848916 & 3034\\ +\hline +2017 & C2T0 & Summer & 68.153952 & 941\\ +\hline +2017 & C2T0 & Winter & 102.823924 & 1841\\ +\hline +2017 & E1L5 & Spring & 83.343390 & 7289\\ +\hline +2017 & E1L5 & Summer & 68.625712 & 2226\\ +\hline +2017 & E1L5 & Winter & 116.024016 & 4872\\ +\hline +2017 & E2L5 & Spring & 10.333572 & 330\\ +\hline +2017 & E2L5 & Summer & 8.378103 & 61\\ +\hline +2017 & E2L5 & Winter & 13.210987 & 279\\ +\hline +2017 & E3L5 & Spring & 45.101418 & 4583\\ +\hline +2017 & E3L5 & Summer & 35.520381 & 1284\\ +\hline +2017 & E3L5 & Winter & 60.735464 & 3343\\ +\hline +\end{tabular} +\end{table} + +Histograms of kW by season and feeder\ldots{} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{ggplot2}\OperatorTok{::}\KeywordTok{ggplot}\NormalTok{(dt, }\KeywordTok{aes}\NormalTok{(}\DataTypeTok{x =}\NormalTok{ kW)) }\OperatorTok{+} +\StringTok{ }\KeywordTok{geom_histogram}\NormalTok{(}\DataTypeTok{binwidth =} \DecValTok{5}\NormalTok{) }\OperatorTok{+} +\StringTok{ }\KeywordTok{facet_grid}\NormalTok{(sub_region }\OperatorTok{~}\StringTok{ }\NormalTok{season)} +\end{Highlighting} +\end{Shaded} + +\includegraphics{cleaningFeederData_files/figure-latex/histo-1.pdf} + +Demand profiles of kW by season and feeder and day of the week\ldots{} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{plotDT <-}\StringTok{ }\NormalTok{dt[, .(}\DataTypeTok{meankW =} \KeywordTok{mean}\NormalTok{(kW)), keyby =}\StringTok{ }\NormalTok{.(rTime, season, sub_region, rDoW)]} +\NormalTok{ggplot2}\OperatorTok{::}\KeywordTok{ggplot}\NormalTok{(plotDT, }\KeywordTok{aes}\NormalTok{(}\DataTypeTok{x =}\NormalTok{ rTime, }\DataTypeTok{y =}\NormalTok{ meankW, }\DataTypeTok{colour =}\NormalTok{ season)) }\OperatorTok{+} +\StringTok{ }\KeywordTok{geom_line}\NormalTok{() }\OperatorTok{+} +\StringTok{ }\KeywordTok{facet_grid}\NormalTok{(sub_region }\OperatorTok{~}\StringTok{ }\NormalTok{rDoW)} +\end{Highlighting} +\end{Shaded} + +\includegraphics{cleaningFeederData_files/figure-latex/kwProfiles-1.pdf} + +Is that what we expect? + +\hypertarget{test-for-missing}{% +\section{Test for missing}\label{test-for-missing}} + +Can we see missing data? + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{ggplot2}\OperatorTok{::}\KeywordTok{ggplot}\NormalTok{(dt, }\KeywordTok{aes}\NormalTok{(}\DataTypeTok{x =}\NormalTok{ rDate, }\DataTypeTok{y =}\NormalTok{ rTime, }\DataTypeTok{fill =}\NormalTok{ kW)) }\OperatorTok{+} +\StringTok{ }\KeywordTok{geom_tile}\NormalTok{() }\OperatorTok{+} +\StringTok{ }\KeywordTok{facet_grid}\NormalTok{(sub_region }\OperatorTok{~}\StringTok{ }\NormalTok{.) }\OperatorTok{+} +\StringTok{ }\KeywordTok{scale_fill_viridis_c}\NormalTok{() }\OperatorTok{+} +\StringTok{ }\KeywordTok{labs}\NormalTok{(}\DataTypeTok{caption =} \StringTok{"All kW values by feeder, time (y) and date (x)"}\NormalTok{)} +\end{Highlighting} +\end{Shaded} + +\includegraphics{cleaningFeederData_files/figure-latex/missingVis-1.pdf} + +oooh. + +Try aggregating\ldots{} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{dt[, rHour }\OperatorTok{:}\ErrorTok{=}\StringTok{ }\NormalTok{lubridate}\OperatorTok{::}\KeywordTok{hour}\NormalTok{(rDateTime)]} +\NormalTok{plotDT <-}\StringTok{ }\NormalTok{dt[, .(}\DataTypeTok{mean_kW =} \KeywordTok{mean}\NormalTok{(kW),} + \DataTypeTok{nObs =}\NormalTok{ .N), keyby =}\StringTok{ }\NormalTok{.(rHour, rDate, sub_region)]} +\NormalTok{ggplot2}\OperatorTok{::}\KeywordTok{ggplot}\NormalTok{(plotDT, }\KeywordTok{aes}\NormalTok{(}\DataTypeTok{x =}\NormalTok{ rDate, }\DataTypeTok{y =}\NormalTok{ rHour, }\DataTypeTok{fill =}\NormalTok{ mean_kW)) }\OperatorTok{+} +\StringTok{ }\KeywordTok{geom_tile}\NormalTok{() }\OperatorTok{+} +\StringTok{ }\KeywordTok{scale_fill_viridis_c}\NormalTok{() }\OperatorTok{+} +\StringTok{ }\KeywordTok{facet_grid}\NormalTok{(sub_region }\OperatorTok{~}\StringTok{ }\NormalTok{.) }\OperatorTok{+} +\StringTok{ }\KeywordTok{labs}\NormalTok{(}\DataTypeTok{caption =} \StringTok{"Mean kW per hour"}\NormalTok{)} +\end{Highlighting} +\end{Shaded} + +\includegraphics{cleaningFeederData_files/figure-latex/aggVis-1.pdf} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{ggplot2}\OperatorTok{::}\KeywordTok{ggplot}\NormalTok{(plotDT, }\KeywordTok{aes}\NormalTok{(}\DataTypeTok{x =}\NormalTok{ rDate, }\DataTypeTok{y =}\NormalTok{ rHour, }\DataTypeTok{fill =}\NormalTok{ nObs)) }\OperatorTok{+} +\StringTok{ }\KeywordTok{geom_tile}\NormalTok{() }\OperatorTok{+} +\StringTok{ }\KeywordTok{scale_fill_viridis_c}\NormalTok{() }\OperatorTok{+} +\StringTok{ }\KeywordTok{facet_grid}\NormalTok{(sub_region }\OperatorTok{~}\StringTok{ }\NormalTok{.) }\OperatorTok{+} +\StringTok{ }\KeywordTok{labs}\NormalTok{(}\DataTypeTok{caption =} \StringTok{"Number of obs per hour"}\NormalTok{)} +\end{Highlighting} +\end{Shaded} + +\includegraphics{cleaningFeederData_files/figure-latex/aggVis-2.pdf} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{ggplot2}\OperatorTok{::}\KeywordTok{ggplot}\NormalTok{(plotDT, }\KeywordTok{aes}\NormalTok{(}\DataTypeTok{x =}\NormalTok{ nObs, }\DataTypeTok{y =}\NormalTok{ mean_kW, }\DataTypeTok{group =}\NormalTok{ nObs)) }\OperatorTok{+} +\StringTok{ }\KeywordTok{geom_boxplot}\NormalTok{() }\OperatorTok{+} +\StringTok{ }\KeywordTok{facet_grid}\NormalTok{(.}\OperatorTok{~}\NormalTok{sub_region) }\OperatorTok{+} +\StringTok{ }\KeywordTok{labs}\NormalTok{(}\DataTypeTok{caption =} \StringTok{"mean kW per hour by number of observations contributing"}\NormalTok{)} +\end{Highlighting} +\end{Shaded} + +\includegraphics{cleaningFeederData_files/figure-latex/aggVis-3.pdf} + +\hypertarget{which-days-have-the-least-missing}{% +\section{Which days have the `least' missing?}\label{which-days-have-the-least-missing}} + +This is quite tricky as we may have completely missing dateTimes. But we can test for this by counting the number of observations per dateTime and then seeing if the dateTimes are contiguous. + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{dateTimesDT <-}\StringTok{ }\NormalTok{dt[, .(}\DataTypeTok{nFeeders =} \KeywordTok{uniqueN}\NormalTok{(sub_region)), keyby =}\StringTok{ }\NormalTok{.(rDateTime, rTime, rDate, season)] }\CommentTok{# keep season} +\NormalTok{dateTimesDT[, dtDiff }\OperatorTok{:}\ErrorTok{=}\StringTok{ }\NormalTok{rDateTime }\OperatorTok{-}\StringTok{ }\KeywordTok{shift}\NormalTok{(rDateTime)] }\CommentTok{# should be 15 mins} + + +\KeywordTok{summary}\NormalTok{(dateTimesDT)} +\end{Highlighting} +\end{Shaded} + +\begin{verbatim} +## rDateTime rTime rDate +## Min. :2017-01-01 00:00:00 Length:15673 Min. :2017-01-01 +## 1st Qu.:2017-02-13 11:30:00 Class1:hms 1st Qu.:2017-02-13 +## Median :2017-03-29 03:15:00 Class2:difftime Median :2017-03-29 +## Mean :2017-03-30 10:30:58 Mode :numeric Mean :2017-03-29 +## 3rd Qu.:2017-05-14 00:15:00 3rd Qu.:2017-05-14 +## Max. :2017-06-30 23:45:00 Max. :2017-06-30 +## season nFeeders dtDiff +## Spring:7954 Min. :1.000 Length:15673 +## Summer:2415 1st Qu.:1.000 Class :difftime +## Autumn: 0 Median :2.000 Mode :numeric +## Winter:5304 Mean :2.315 +## 3rd Qu.:3.000 +## Max. :5.000 +\end{verbatim} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{ggplot2}\OperatorTok{::}\KeywordTok{ggplot}\NormalTok{(dateTimesDT, }\KeywordTok{aes}\NormalTok{(}\DataTypeTok{x =}\NormalTok{ rDate, }\DataTypeTok{y =}\NormalTok{ rTime, }\DataTypeTok{fill =}\NormalTok{ nFeeders)) }\OperatorTok{+} +\StringTok{ }\KeywordTok{geom_tile}\NormalTok{() }\OperatorTok{+} +\StringTok{ }\KeywordTok{scale_fill_viridis_c}\NormalTok{() }\OperatorTok{+} +\StringTok{ }\KeywordTok{labs}\NormalTok{(}\DataTypeTok{caption =} \StringTok{"Number of unique feeders in each dateTime"}\NormalTok{)} +\end{Highlighting} +\end{Shaded} + +\includegraphics{cleaningFeederData_files/figure-latex/testDateTimes-1.pdf} + +Well we clearly have some dateTimes where we have no data \emph{at all}! + +Are there time of day patterns? It looks like it\ldots{} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{plotDT <-}\StringTok{ }\NormalTok{dateTimesDT[, .(}\DataTypeTok{meanN =} \KeywordTok{mean}\NormalTok{(nFeeders)), keyby =}\StringTok{ }\NormalTok{.(rTime, season)]} +\NormalTok{ggplot2}\OperatorTok{::}\KeywordTok{ggplot}\NormalTok{(plotDT, }\KeywordTok{aes}\NormalTok{(}\DataTypeTok{y =}\NormalTok{ meanN, }\DataTypeTok{x =}\NormalTok{ rTime, }\DataTypeTok{colour =}\NormalTok{ season)) }\OperatorTok{+} +\StringTok{ }\KeywordTok{geom_line}\NormalTok{() }\OperatorTok{+} +\StringTok{ }\KeywordTok{labs}\NormalTok{(}\DataTypeTok{y =} \StringTok{"Mean n feeders reporting"}\NormalTok{,} + \DataTypeTok{caption =} \StringTok{"Mean n feeders by time of day"}\NormalTok{)} +\end{Highlighting} +\end{Shaded} + +\includegraphics{cleaningFeederData_files/figure-latex/missingProfiles-1.pdf} + +So\ldots{} + +I find it distinctly odd that: + +\begin{itemize} +\tightlist +\item + we appear to have the most feeders reporting data at `peak' times +\item + we have a lot of missing dateTimes between 00:30 and 05:00 +\end{itemize} + +If the monitors were set to only collect data when the power (or Wh in a given time frame) was above a given threshold then it would look like this\ldots{} + +\hypertarget{runtime}{% +\section{Runtime}\label{runtime}} + +Analysis completed in 15.32 seconds ( 0.26 minutes) using \href{https://cran.r-project.org/package=knitr}{knitr} in \href{http://www.rstudio.com}{RStudio} with R version 3.6.3 (2020-02-29) running on x86\_64-apple-darwin15.6.0. + +\hypertarget{r-environment}{% +\section{R environment}\label{r-environment}} + +\hypertarget{r-packages-used}{% +\subsection{R packages used}\label{r-packages-used}} + +\begin{itemize} +\tightlist +\item + base R (R Core Team 2016) +\item + bookdown (Xie 2016a) +\item + data.table (Dowle et al. 2015) +\item + ggplot2 (Wickham 2009) +\item + kableExtra (Zhu 2018) +\item + knitr (Xie 2016b) +\item + lubridate (Grolemund and Wickham 2011) +\item + rmarkdown (Allaire et al. 2018) +\item + skimr (Arino de la Rubia et al. 2017) +\end{itemize} + +\hypertarget{session-info}{% +\subsection{Session info}\label{session-info}} + +\begin{verbatim} +## R version 3.6.3 (2020-02-29) +## Platform: x86_64-apple-darwin15.6.0 (64-bit) +## Running under: macOS Catalina 10.15.5 +## +## Matrix products: default +## BLAS: /Library/Frameworks/R.framework/Versions/3.6/Resources/lib/libRblas.0.dylib +## LAPACK: /Library/Frameworks/R.framework/Versions/3.6/Resources/lib/libRlapack.dylib +## +## locale: +## [1] en_NZ.UTF-8/en_NZ.UTF-8/en_NZ.UTF-8/C/en_NZ.UTF-8/en_NZ.UTF-8 +## +## attached base packages: +## [1] stats graphics grDevices utils datasets methods base +## +## other attached packages: +## [1] skimr_2.1 kableExtra_1.1.0 ggplot2_3.3.0 hms_0.5.3 +## [5] lubridate_1.7.4 data.table_1.12.8 +## +## loaded via a namespace (and not attached): +## [1] Rcpp_1.0.4 compiler_3.6.3 pillar_1.4.3 base64enc_0.1-3 +## [5] tools_3.6.3 digest_0.6.25 jsonlite_1.6.1 viridisLite_0.3.0 +## [9] evaluate_0.14 lifecycle_0.2.0 tibble_2.1.3 gtable_0.3.0 +## [13] pkgconfig_2.0.3 rlang_0.4.5 rstudioapi_0.11 yaml_2.2.1 +## [17] xfun_0.12 repr_1.1.0 xml2_1.2.5 httr_1.4.1 +## [21] withr_2.1.2 stringr_1.4.0 dplyr_0.8.5 knitr_1.28 +## [25] vctrs_0.2.4 webshot_0.5.2 grid_3.6.3 tidyselect_1.0.0 +## [29] glue_1.3.2 R6_2.4.1 rmarkdown_2.1 bookdown_0.18 +## [33] farver_2.0.3 purrr_0.3.3 readr_1.3.1 magrittr_1.5 +## [37] scales_1.1.0 htmltools_0.4.0 rvest_0.3.5 assertthat_0.2.1 +## [41] colorspace_1.4-1 labeling_0.3 stringi_1.4.6 munsell_0.5.0 +## [45] crayon_1.3.4 +\end{verbatim} + +\hypertarget{the-data-cleaning-code}{% +\section{The data cleaning code}\label{the-data-cleaning-code}} + +\begin{enumerate} +\def\labelenumi{(\alph{enumi})} +\setcounter{enumi}{2} +\tightlist +\item + Mikey Harper :-) +\end{enumerate} + +Starts here: + +Scripts used clean and merge substation data. + +\hypertarget{input-files}{% +\subsection{Input files}\label{input-files}} + +Analysis will first look at the primary data. There are different types of files which refer to different paramters. Different search terms are used to extract these: + +\begin{Shaded} +\begin{Highlighting}[] +\CommentTok{# Find files with AMPS. Exclude files which contain DI~CO} +\NormalTok{files_AMPS <-}\StringTok{ }\KeywordTok{list.files}\NormalTok{(}\StringTok{"../Primary"}\NormalTok{, }\DataTypeTok{recursive =}\NormalTok{ T, }\DataTypeTok{pattern =} \StringTok{"~AMPS"}\NormalTok{, }\DataTypeTok{full.names =}\NormalTok{ T) }\OperatorTok{%>%} +\StringTok{ }\NormalTok{.[}\OperatorTok{!}\NormalTok{stringr}\OperatorTok{::}\KeywordTok{str_detect}\NormalTok{ (., }\StringTok{"DI~CO"}\NormalTok{)]} + +\NormalTok{files_AMPS} +\end{Highlighting} +\end{Shaded} + +\hypertarget{process-amps}{% +\subsection{Process Amps}\label{process-amps}} + +\begin{Shaded} +\begin{Highlighting}[] +\CommentTok{# Show a sample} +\NormalTok{fileSelect <-}\StringTok{ }\NormalTok{files_AMPS[}\DecValTok{4}\NormalTok{]} +\KeywordTok{head}\NormalTok{(}\KeywordTok{read_csv}\NormalTok{(fileSelect, }\DataTypeTok{skip =} \DecValTok{3}\NormalTok{))} +\end{Highlighting} +\end{Shaded} + +Again a function is used to do all the processing on the input CSVs. This is slightly amended from the \texttt{processkV} function. + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{processAMPS <-}\StringTok{ }\ControlFlowTok{function}\NormalTok{(filePath, }\DataTypeTok{databaseCon =}\NormalTok{ con)\{} + + \KeywordTok{message}\NormalTok{(}\StringTok{"Processing "}\NormalTok{, filePath)} + + \CommentTok{# 1st Level} +\NormalTok{ dirName_}\DecValTok{1}\NormalTok{ <-}\StringTok{ }\NormalTok{filePath }\OperatorTok{%>%}\StringTok{ } +\StringTok{ }\KeywordTok{dirname}\NormalTok{() }\OperatorTok{%>%}\StringTok{ } +\StringTok{ }\NormalTok{basename} + + \CommentTok{# 2nd Level} +\NormalTok{ dirName_}\DecValTok{2}\NormalTok{ <-}\StringTok{ }\NormalTok{filePath }\OperatorTok{%>%}\StringTok{ } +\StringTok{ }\KeywordTok{dirname}\NormalTok{() }\OperatorTok{%>%}\StringTok{ } +\StringTok{ }\KeywordTok{dirname}\NormalTok{() }\OperatorTok{%>%}\StringTok{ } +\StringTok{ }\NormalTok{basename} + + \ControlFlowTok{if}\NormalTok{ (dirName_}\DecValTok{2} \OperatorTok{==}\StringTok{ "Primary"}\NormalTok{)\{} +\NormalTok{ dirName_}\DecValTok{2}\NormalTok{ <-}\StringTok{ }\NormalTok{dirName_}\DecValTok{1} +\NormalTok{ dirName_}\DecValTok{1}\NormalTok{ <-}\StringTok{ ""} +\NormalTok{ \}} + + \CommentTok{# Load the CSV. There were some tab seperated files which are saved as CSVs, which confuse the search. There if the data is loaded incorrectly (only having a single column), the code will try and load it as a TSV.} +\NormalTok{ dataLoaded <-}\StringTok{ }\KeywordTok{suppressWarnings}\NormalTok{(}\KeywordTok{read_csv}\NormalTok{(filePath, }\DataTypeTok{skip =} \DecValTok{3}\NormalTok{, }\DataTypeTok{col_types =} \KeywordTok{cols}\NormalTok{(}\DataTypeTok{Value =} \KeywordTok{col_number}\NormalTok{())))} + \ControlFlowTok{if}\NormalTok{(}\KeywordTok{ncol}\NormalTok{(dataLoaded) }\OperatorTok{==}\StringTok{ }\DecValTok{1}\NormalTok{)\{} +\NormalTok{ dataLoaded <-}\StringTok{ }\KeywordTok{suppressWarnings}\NormalTok{(}\KeywordTok{read_tsv}\NormalTok{(filePath, }\DataTypeTok{skip =} \DecValTok{3}\NormalTok{, }\DataTypeTok{col_types =} \KeywordTok{cols}\NormalTok{()))} +\NormalTok{ \}} + + \CommentTok{# Reformat data} +\NormalTok{ dataLoaded <-} +\StringTok{ }\NormalTok{dataLoaded }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{mutate_at}\NormalTok{(}\KeywordTok{vars}\NormalTok{(Time), }\ControlFlowTok{function}\NormalTok{(x)\{}\KeywordTok{gsub}\NormalTok{(}\StringTok{'[^ -~]'}\NormalTok{, }\StringTok{''}\NormalTok{, x)\}) }\OperatorTok{%>%}\StringTok{ }\CommentTok{# Remove invalid UTF characters} +\StringTok{ }\KeywordTok{mutate}\NormalTok{(}\DataTypeTok{Time =}\NormalTok{ lubridate}\OperatorTok{::}\KeywordTok{dmy_hms}\NormalTok{(Time),} + \DataTypeTok{Time =}\NormalTok{ lubridate}\OperatorTok{::}\KeywordTok{floor_date}\NormalTok{(Time, }\DataTypeTok{unit =} \StringTok{"15 minutes"}\NormalTok{)) }\OperatorTok{%>%}\StringTok{ } +\StringTok{ }\KeywordTok{group_by}\NormalTok{(Time) }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{summarise}\NormalTok{(}\DataTypeTok{Value =} \KeywordTok{mean}\NormalTok{(Value, }\DataTypeTok{na.rm =}\NormalTok{ T)) }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{mutate}\NormalTok{(}\DataTypeTok{region =}\NormalTok{ dirName_}\DecValTok{2}\NormalTok{,} + \DataTypeTok{sub_region =}\NormalTok{ dirName_}\DecValTok{1} +\NormalTok{ )} + + \CommentTok{# There are some datasets which contain no values, whch can cause errors in running} + \CommentTok{# If this happens, return NULL} + \ControlFlowTok{if}\NormalTok{(}\KeywordTok{is.character}\NormalTok{(dataLoaded}\OperatorTok{$}\NormalTok{Value)) }\KeywordTok{return}\NormalTok{(}\OtherTok{NULL}\NormalTok{)} + + \KeywordTok{return}\NormalTok{(dataLoaded)} +\NormalTok{\}} +\end{Highlighting} +\end{Shaded} + +Run the function below: + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{Amps <-}\StringTok{ }\NormalTok{purrr}\OperatorTok{::}\KeywordTok{map_df}\NormalTok{(files_AMPS, processAMPS)} +\end{Highlighting} +\end{Shaded} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{Amps_stats <-}\StringTok{ }\NormalTok{Amps }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{group_by}\NormalTok{(region) }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{summarise}\NormalTok{(}\DataTypeTok{mean =}\NormalTok{ (}\KeywordTok{mean}\NormalTok{(Value, }\DataTypeTok{na.rm =}\NormalTok{ T)),} + \DataTypeTok{n =} \KeywordTok{n}\NormalTok{(),} + \DataTypeTok{sd =} \KeywordTok{sd}\NormalTok{(Value, }\DataTypeTok{na.rm =}\NormalTok{ T),} + \DataTypeTok{var =} \KeywordTok{var}\NormalTok{(Value, }\DataTypeTok{na.rm =}\NormalTok{ T))} + +\NormalTok{Amps_stats} + +\NormalTok{readr}\OperatorTok{::}\KeywordTok{write_csv}\NormalTok{(Amps_stats, }\DataTypeTok{path =} \StringTok{"../Amps_stats.csv"}\NormalTok{)} +\end{Highlighting} +\end{Shaded} + +\begin{Shaded} +\begin{Highlighting}[] +\KeywordTok{ggplot}\NormalTok{(Amps) }\OperatorTok{+} +\StringTok{ }\KeywordTok{geom_point}\NormalTok{(}\KeywordTok{aes}\NormalTok{(}\DataTypeTok{x =}\NormalTok{ Time, }\DataTypeTok{y =}\NormalTok{ Value, }\DataTypeTok{colour =}\NormalTok{ region)) }\OperatorTok{+} +\StringTok{ }\KeywordTok{facet_grid}\NormalTok{(region}\OperatorTok{~}\NormalTok{., }\DataTypeTok{scales =} \StringTok{"free_y"}\NormalTok{) }\OperatorTok{+} +\StringTok{ }\KeywordTok{labs}\NormalTok{(}\DataTypeTok{title =} \StringTok{"Cleaned data for Amps"}\NormalTok{)} +\end{Highlighting} +\end{Shaded} + +\hypertarget{processing-data}{% +\subsection{Processing data}\label{processing-data}} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{readr}\OperatorTok{::}\KeywordTok{write_csv}\NormalTok{(Amps, }\DataTypeTok{path =} \StringTok{"amps_all_substations.csv"}\NormalTok{)} +\end{Highlighting} +\end{Shaded} + +\begin{Shaded} +\begin{Highlighting}[] +\KeywordTok{library}\NormalTok{(odbc)} + +\KeywordTok{library}\NormalTok{(DBI)} +\CommentTok{# Create an ephemeral in-memory RSQLite database} +\NormalTok{con <-}\StringTok{ }\KeywordTok{dbConnect}\NormalTok{(RSQLite}\OperatorTok{::}\KeywordTok{SQLite}\NormalTok{(), }\StringTok{"amps.sqlite"}\NormalTok{)} + +\KeywordTok{dbListTables}\NormalTok{(con)} + + +\KeywordTok{dbWriteTable}\NormalTok{(con, }\StringTok{"amps"}\NormalTok{, Amps)} +\KeywordTok{dbListTables}\NormalTok{(con)} +\end{Highlighting} +\end{Shaded} + +\hypertarget{querying-the-data}{% +\subsection{Querying the data}\label{querying-the-data}} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{con <-}\StringTok{ }\KeywordTok{dbConnect}\NormalTok{(RSQLite}\OperatorTok{::}\KeywordTok{SQLite}\NormalTok{(), }\StringTok{"amps.sqlite"}\NormalTok{)} + + +\KeywordTok{library}\NormalTok{(dbplyr)} + +\NormalTok{Amps_db <-}\StringTok{ }\KeywordTok{tbl}\NormalTok{(con, }\StringTok{"amps"}\NormalTok{)} + + +\NormalTok{flights_db }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{group_by}\NormalTok{(region) }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{summarise}\NormalTok{(}\DataTypeTok{mean =}\NormalTok{ (}\KeywordTok{mean}\NormalTok{(Value, }\DataTypeTok{na.rm =}\NormalTok{ T)),} + \DataTypeTok{n =} \KeywordTok{n}\NormalTok{(),} + \DataTypeTok{sd =} \KeywordTok{sd}\NormalTok{(Value, }\DataTypeTok{na.rm =}\NormalTok{ T),} + \DataTypeTok{var =} \KeywordTok{var}\NormalTok{(Value, }\DataTypeTok{na.rm =}\NormalTok{ T))} +\end{Highlighting} +\end{Shaded} + +\hypertarget{round-to-nearest-n-minutes}{% +\subsection{Round to Nearest N minutes}\label{round-to-nearest-n-minutes}} + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{processAMPS_5mins <-}\StringTok{ }\ControlFlowTok{function}\NormalTok{(filePath)\{} + + \KeywordTok{message}\NormalTok{(}\StringTok{"Processing "}\NormalTok{, filePath)} + + \CommentTok{# 1st Level} +\NormalTok{ dirName_}\DecValTok{1}\NormalTok{ <-}\StringTok{ }\NormalTok{filePath }\OperatorTok{%>%}\StringTok{ } +\StringTok{ }\KeywordTok{dirname}\NormalTok{() }\OperatorTok{%>%}\StringTok{ } +\StringTok{ }\NormalTok{basename} + + \CommentTok{# 2nd Level} +\NormalTok{ dirName_}\DecValTok{2}\NormalTok{ <-}\StringTok{ }\NormalTok{filePath }\OperatorTok{%>%}\StringTok{ } +\StringTok{ }\KeywordTok{dirname}\NormalTok{() }\OperatorTok{%>%}\StringTok{ } +\StringTok{ }\KeywordTok{dirname}\NormalTok{() }\OperatorTok{%>%}\StringTok{ } +\StringTok{ }\NormalTok{basename} + + \ControlFlowTok{if}\NormalTok{ (dirName_}\DecValTok{2} \OperatorTok{==}\StringTok{ "Primary"}\NormalTok{)\{} +\NormalTok{ dirName_}\DecValTok{2}\NormalTok{ <-}\StringTok{ }\NormalTok{dirName_}\DecValTok{1} +\NormalTok{ dirName_}\DecValTok{1}\NormalTok{ <-}\StringTok{ ""} +\NormalTok{ \}} + + \CommentTok{# Load the CSV. There were some tab seperated files which are saved as CSVs, which confuse the search. There if the data is loaded incorrectly (only having a single column), the code will try and load it as a TSV.} +\NormalTok{ dataLoaded <-}\StringTok{ }\KeywordTok{suppressWarnings}\NormalTok{(}\KeywordTok{read_csv}\NormalTok{(filePath, }\DataTypeTok{skip =} \DecValTok{3}\NormalTok{, }\DataTypeTok{col_types =} \KeywordTok{cols}\NormalTok{()))} + \ControlFlowTok{if}\NormalTok{(}\KeywordTok{ncol}\NormalTok{(dataLoaded) }\OperatorTok{==}\StringTok{ }\DecValTok{1}\NormalTok{)\{} +\NormalTok{ dataLoaded <-}\StringTok{ }\KeywordTok{suppressWarnings}\NormalTok{(}\KeywordTok{read_tsv}\NormalTok{(filePath, }\DataTypeTok{skip =} \DecValTok{3}\NormalTok{, }\DataTypeTok{col_types =} \KeywordTok{cols}\NormalTok{()))} +\NormalTok{ \}} + + \CommentTok{# Reformat data} +\NormalTok{ dataLoaded <-} +\StringTok{ }\NormalTok{dataLoaded }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{mutate_at}\NormalTok{(}\KeywordTok{vars}\NormalTok{(Time), }\ControlFlowTok{function}\NormalTok{(x)\{}\KeywordTok{gsub}\NormalTok{(}\StringTok{'[^ -~]'}\NormalTok{, }\StringTok{''}\NormalTok{, x)\}) }\OperatorTok{%>%}\StringTok{ }\CommentTok{# Remove invalid UTF characters} +\StringTok{ }\KeywordTok{mutate}\NormalTok{(}\DataTypeTok{Time =}\NormalTok{ lubridate}\OperatorTok{::}\KeywordTok{dmy_hms}\NormalTok{(Time),} + \DataTypeTok{region =}\NormalTok{ dirName_}\DecValTok{2}\NormalTok{,} + \DataTypeTok{sub_region =}\NormalTok{ dirName_}\DecValTok{1}\NormalTok{,} + \DataTypeTok{code =} \KeywordTok{paste}\NormalTok{(region, sub_region, }\DataTypeTok{sep =} \StringTok{"_"}\NormalTok{),} +\NormalTok{ ) }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{mutate}\NormalTok{(}\DataTypeTok{Time =}\NormalTok{ lubridate}\OperatorTok{::}\KeywordTok{floor_date}\NormalTok{(Time, }\DataTypeTok{unit =} \StringTok{"5 minutes"}\NormalTok{)) }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{group_by}\NormalTok{(Time, region, code) }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{summarise}\NormalTok{(}\DataTypeTok{Value =} \KeywordTok{mean}\NormalTok{(Value)) }\OperatorTok{%>%} +\StringTok{ }\KeywordTok{arrange}\NormalTok{(Time)} + + \CommentTok{# There are some datasets which contain no values, whch can cause errors in running} + \CommentTok{# If this happens, return NULL} + \ControlFlowTok{if}\NormalTok{(}\KeywordTok{is.character}\NormalTok{(dataLoaded}\OperatorTok{$}\NormalTok{Value)) }\KeywordTok{return}\NormalTok{(}\OtherTok{NULL}\NormalTok{)} + + \CommentTok{# Returns the loaded and cleaned dataframe} + \KeywordTok{return}\NormalTok{(dataLoaded)} +\NormalTok{\}} +\end{Highlighting} +\end{Shaded} + +Nearest 5 minutes: + +\begin{Shaded} +\begin{Highlighting}[] +\NormalTok{Amps_5mins <<-}\StringTok{ }\NormalTok{purrr}\OperatorTok{::}\KeywordTok{map_df}\NormalTok{(files_AMPS[}\DecValTok{1}\OperatorTok{:}\DecValTok{4}\NormalTok{], processAMPS_5mins)} +\end{Highlighting} +\end{Shaded} + +\hypertarget{references}{% +\section*{References}\label{references}} +\addcontentsline{toc}{section}{References} + +\hypertarget{refs}{} +\leavevmode\hypertarget{ref-rmarkdown}{}% +Allaire, JJ, Yihui Xie, Jonathan McPherson, Javier Luraschi, Kevin Ushey, Aron Atkins, Hadley Wickham, Joe Cheng, and Winston Chang. 2018. \emph{Rmarkdown: Dynamic Documents for R}. \url{https://CRAN.R-project.org/package=rmarkdown}. + +\leavevmode\hypertarget{ref-skimr}{}% +Arino de la Rubia, Eduardo, Hao Zhu, Shannon Ellis, Elin Waring, and Michael Quinn. 2017. \emph{Skimr: Skimr}. \url{https://github.com/ropenscilabs/skimr}. + +\leavevmode\hypertarget{ref-data.table}{}% +Dowle, M, A Srinivasan, T Short, S Lianoglou with contributions from R Saporta, and E Antonyan. 2015. \emph{Data.table: Extension of Data.frame}. \url{https://CRAN.R-project.org/package=data.table}. + +\leavevmode\hypertarget{ref-lubridate}{}% +Grolemund, Garrett, and Hadley Wickham. 2011. ``Dates and Times Made Easy with lubridate.'' \emph{Journal of Statistical Software} 40 (3): 1--25. \url{http://www.jstatsoft.org/v40/i03/}. + +\leavevmode\hypertarget{ref-baseR}{}% +R Core Team. 2016. \emph{R: A Language and Environment for Statistical Computing}. Vienna, Austria: R Foundation for Statistical Computing. \url{https://www.R-project.org/}. + +\leavevmode\hypertarget{ref-ggplot2}{}% +Wickham, Hadley. 2009. \emph{Ggplot2: Elegant Graphics for Data Analysis}. Springer-Verlag New York. \url{http://ggplot2.org}. + +\leavevmode\hypertarget{ref-bookdown}{}% +Xie, Yihui. 2016a. \emph{Bookdown: Authoring Books and Technical Documents with R Markdown}. Boca Raton, Florida: Chapman; Hall/CRC. \url{https://github.com/rstudio/bookdown}. + +\leavevmode\hypertarget{ref-knitr}{}% +---------. 2016b. \emph{Knitr: A General-Purpose Package for Dynamic Report Generation in R}. \url{https://CRAN.R-project.org/package=knitr}. + +\leavevmode\hypertarget{ref-kableExtra}{}% +Zhu, Hao. 2018. \emph{KableExtra: Construct Complex Table with 'Kable' and Pipe Syntax}. \url{https://CRAN.R-project.org/package=kableExtra}. + +\end{document} diff --git a/isleOfWight/cleaningFeederData_files/figure-latex/aggVis-1.pdf b/isleOfWight/cleaningFeederData_files/figure-latex/aggVis-1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..70c3fe469a85c0047a3891020c334512f2925d4a Binary files /dev/null and b/isleOfWight/cleaningFeederData_files/figure-latex/aggVis-1.pdf differ diff --git a/isleOfWight/cleaningFeederData_files/figure-latex/aggVis-2.pdf b/isleOfWight/cleaningFeederData_files/figure-latex/aggVis-2.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e830bf37e360200657dc8433d87f26b2599084d9 Binary files /dev/null and b/isleOfWight/cleaningFeederData_files/figure-latex/aggVis-2.pdf differ diff --git a/isleOfWight/cleaningFeederData_files/figure-latex/aggVis-3.pdf b/isleOfWight/cleaningFeederData_files/figure-latex/aggVis-3.pdf new file mode 100644 index 0000000000000000000000000000000000000000..abfeb6a4a860a510195a5c344c2b1a3fc44cfae3 Binary files /dev/null and b/isleOfWight/cleaningFeederData_files/figure-latex/aggVis-3.pdf differ diff --git a/isleOfWight/cleaningFeederData_files/figure-latex/histo-1.pdf b/isleOfWight/cleaningFeederData_files/figure-latex/histo-1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7dbb281d7a0b3e930badbb8a25034b2e2c414cea Binary files /dev/null and b/isleOfWight/cleaningFeederData_files/figure-latex/histo-1.pdf differ diff --git a/isleOfWight/cleaningFeederData_files/figure-latex/kwProfiles-1.pdf b/isleOfWight/cleaningFeederData_files/figure-latex/kwProfiles-1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d89392c949b113ef581fe8941268322b86c7c0f3 Binary files /dev/null and b/isleOfWight/cleaningFeederData_files/figure-latex/kwProfiles-1.pdf differ diff --git a/isleOfWight/cleaningFeederData_files/figure-latex/missingProfiles-1.pdf b/isleOfWight/cleaningFeederData_files/figure-latex/missingProfiles-1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..01e96767fb9ce23f7144647f8a319a2c8cf5f7b3 Binary files /dev/null and b/isleOfWight/cleaningFeederData_files/figure-latex/missingProfiles-1.pdf differ diff --git a/isleOfWight/cleaningFeederData_files/figure-latex/missingVis-1.pdf b/isleOfWight/cleaningFeederData_files/figure-latex/missingVis-1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..dbc8678483b40a30a3b0759a93ee8e1e0e23e11b Binary files /dev/null and b/isleOfWight/cleaningFeederData_files/figure-latex/missingVis-1.pdf differ diff --git a/isleOfWight/cleaningFeederData_files/figure-latex/testDateTimes-1.pdf b/isleOfWight/cleaningFeederData_files/figure-latex/testDateTimes-1.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c881d36eac706c6d0270baf581c9f3195545fd13 Binary files /dev/null and b/isleOfWight/cleaningFeederData_files/figure-latex/testDateTimes-1.pdf differ