diff --git a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.Rmd b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.Rmd
index 5c064108a4998a4dca3bf27e566b20413475916a..2aad5941c9ace96574a7a37a4962ddb6cda9e31e 100644
--- a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.Rmd
+++ b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.Rmd
@@ -272,6 +272,14 @@ mtusW6Path <- "~/Data/MTUS/World_6/"
mtusW6aUKEpsF <- paste0(mtusW6Path, "processed/mtusW6aEpsUnifiedDT.csv.gz")
mtusW6aUKSurveyF <- paste0(mtusW6Path, "processed/mtusW6aSurveyUnifiedDT.csv.gz")
syntheticDiaryFile <- paste0(mtusW6Path, "processed/synthMTUSW6aEpsDT.csv.gz")
+
+# NG data
+# data held locally but we could also pull it from:
+# 2005-2010 https://www.nationalgrid.com/sites/default/files/documents/DemandData_2005-2010.csv
+# 2011-2016 https://www.nationalgrid.com/sites/default/files/documents/DemandData_2011-2016.csv
+# 2017 https://www.nationalgrid.com/sites/default/files/documents/DemandData_2017.csv
+
+ngDPath <- "~/Data/UK_National_Grid"
```
# To do
@@ -293,6 +301,111 @@ syntheticDiaryFile <- paste0(mtusW6Path, "processed/synthMTUSW6aEpsDT.csv.gz")
* Most models (MARKAL, TIMES) tend to quantify change of energy demand in the future as average consumption per day/per year (exeption from EST)
* Areas in which time use data has been used longitudinally (not in energy)
+## National Grid system peaks
+
+We use this data to try to examine changing peaks over time. Note that these demand values include _all_ demand, not just households.
+
+```{r load NG demand data}
+
+ng2005_2010DT <- as.data.table(read_csv(paste0(ngDPath,"/DemandData_2005-2010.csv"), progress = FALSE))
+ng2011_2016DT <- as.data.table(read_csv(paste0(ngDPath,"/DemandData_2011-2016.csv"), progress = FALSE))
+ng2017DT <- as.data.table(read_csv(paste0(ngDPath,"/DemandData_2017.csv"), progress = FALSE))
+
+# fix dates - it would be so so so much better if these were proper half hours!
+ng2005_2010DT <- ng2005_2010DT[, r_date := dmy(SETTLEMENT_DATE)] # requires lubridate
+ng2011_2016DT <- ng2011_2016DT[, r_date := dmy(SETTLEMENT_DATE)] # requires lubridate
+ng2017DT <- ng2017DT[, r_date := dmy(SETTLEMENT_DATE)] # requires lubridate
+
+# rbind them to give full dataset
+ngDemandDT <- rbind(ng2005_2010DT,ng2011_2016DT,ng2017DT, fill = TRUE) # fill missing col in first file
+
+# set some dates & times
+ngDemandDT <- ngDemandDT[, r_month := month(r_date, label = TRUE, abbr = TRUE)] # requires lubridate
+ngDemandDT <- ngDemandDT[, r_year := year(r_date)] # requires lubridate
+ngDemandDT <- ngDemandDT[, r_dow := wday(r_date, label = TRUE, abbr = TRUE)] # requires lubridate
+# table(ngDemandDT$r_year, ngDemandDT$r_month)
+# create a half-hour date-time
+ngDemandDT <- ngDemandDT[, r_fractionalHour := (SETTLEMENT_PERIOD/2)-0.5]
+ngDemandDT <- ngDemandDT[, r_hour := floor(r_fractionalHour/1)]
+ngDemandDT <- ngDemandDT[, r_mins := ifelse((r_fractionalHour*10)%%10==0, 0, 30)]
+# this is broken
+# ngDemandDT <- ngDemandDT[, r_halfHour := paste0(r_hour, ":", r_mins)]
+# ngDemandDT <- ngDemandDT[, r_dateTime := strptime(paste0(r_date, " ", r_hour, ":", r_mins), "%y-%m-%d %H:%M" )]
+
+# run some data checks at monthly level
+dataNote <- "Source: National Grid half-hourly demand data (England & Wales) 2005-2017"
+
+tempDT <- ngDemandDT[, .(meanMW = mean(ENGLAND_WALES_DEMAND),
+ nObs = .N), keyby = .(r_year, r_month)]
+
+ggplot(tempDT, aes(x = r_year, y = r_month, fill = nObs)) +
+ geom_tile() +
+ labs(caption = paste0("Data check: number of monthly half hour observatons\n", dataNote),
+ x = "Year",
+ y = "Month")
+
+ggplot(tempDT, aes(x = r_year, y = r_month, fill = meanMW)) +
+ geom_tile() +
+ scale_fill_continuous(low = "green", high = "red") +
+ labs(caption = paste0("Monthly mean MW\n", dataNote),
+ x = "Year",
+ y = "Month")
+
+
+ggplot(tempDT, aes(x = r_month, y = meanMW, colour = r_year, group = r_year)) + # need to fix the fractional years on the legend
+ geom_line() +
+ ylim(0,NA) +
+ labs(caption = paste0("Monthly mean MW\n", dataNote),
+ x = "Month",
+ y = "Year") +
+ theme(legend.title=element_blank()) #http://www.cookbook-r.com/Graphs/Legends_%28ggplot2%29/#hiding-the-legend-title
+
+
+```
+
+The last chart shows a quite substantial drop in demand over the last 10 years... at all times of year.
+
+The next two charts show demand levels in January 2006 & 2016 as mean MW per hour.
+
+```{r compareNGJanuary2006toJanuary2016}
+dataNote <- "Source: National Grid half-hourly demand data (England & Wales) 2006-2016"
+
+extractDT <- ngDemandDT[r_year == 2006 | r_year == 2011 | r_year == 2016]
+
+tempDT <- extractDT[r_dow != "Sat" & r_dow != "Sun", .( meanMW = mean(ENGLAND_WALES_DEMAND)), keyby = .(r_hour, r_year)]
+
+rawPlot <- ggplot(tempDT, aes(x = r_hour, y = meanMW, colour = as.factor(r_year), group = r_year)) + # need to fix the fractional years on the legend
+ geom_line() +
+ #ylim(0,NA) +
+ labs(caption = paste0("Hourly mean MW (weekdays)\n", dataNote),
+ x = "Hour",
+ y = "MW") +
+ theme(legend.title=element_blank()) #http://www.cookbook-r.com/Graphs/Legends_%28ggplot2%29/#hiding-the-legend-title
+
+rawPlot + scale_colour_grey(start = 0.8, end = 0.2)
+
+# normalise by the mean for each year
+mean2006 <- mean(tempDT[r_year == 2006]$meanMW)
+mean2011 <- mean(tempDT[r_year == 2011]$meanMW)
+mean2016 <- mean(tempDT[r_year == 2016]$meanMW)
+
+tempDT <- tempDT[r_year == 2006, normMean := meanMW/mean2006]
+tempDT <- tempDT[r_year == 2011, normMean := meanMW/mean2011]
+tempDT <- tempDT[r_year == 2016, normMean := meanMW/mean2016]
+
+normPlot <- ggplot(tempDT, aes(x = r_hour, y = normMean, colour = as.factor(r_year), group = r_year)) + # need to fix the fractional years on the legend
+ geom_line() +
+ #ylim(0,NA) +
+ labs(caption = paste0("Normalised hourly mean MW (weekdays)\n", dataNote),
+ x = "Hour",
+ y = "MW") +
+ theme(legend.title=element_blank()) #http://www.cookbook-r.com/Graphs/Legends_%28ggplot2%29/#hiding-the-legend-title
+
+normPlot + scale_colour_grey(start = 0.8, end = 0.2)
+
+```
+
+
# Methods
* Details about longitudinal analysis
- NB: this is not `really` longitudinal (following the same individuals over time), it is analysis of a series of cross-sectional samples
@@ -305,6 +418,7 @@ Data used:
* [MTUS World 6]((http://www.timeuse.org/mtus)) - Multinational Timeuse Survey sample for the UK 1974-2005. Note that the most recent MTUS release is W9 but as far as we can tell it has no additional data of use in this paper;
* [UK Time Use Survey 2014-2015](https://discover.ukdataservice.ac.uk/catalogue/?sn=8128) data converted to MTUS format.
+ * [National Grid 2005-2016 England & Wales demand data](https://www.nationalgrid.com/uk/electricity/market-operations-and-data/data-explorer)
The following section loads the [harmonised](https://dataknut.github.io/UK-TU-2014/convertToMTUS/createMTUSFromUKTU2014.html) MTUS & UK TU 2014-2015 data which has then been processed to form a [synthetic half-hour dataset]() that can enable comparisons over time. Much of this output will not be necessary for the paper but is useful detail here to understand what we are doing.
@@ -361,6 +475,9 @@ kable(caption="MTUS 1974-2015 Survay data: age ranges (weighted)",
)
```
+
+
+
## Synthetic halfhour MTUS 1974-2014
Now load the synthetic 'half hour' MTUS 1974 - 2014 data we previously created.
@@ -448,6 +565,7 @@ synthW6aEpsDT <- dt # this will definitely now only have episodes that are from
dt <- NULL # remove to save memory
```
+
# DEMAND Acts
We use this data to code the main and secondary activities into a non-arbitrary but highly aggregated set of 10 DEMAND 'Acts'. This enables us to more easily depict change over time at a coarse level. The aggregated codes are as follows:
diff --git a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.html b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.html
index 93e0fbb1b985215387c499e0445dbcce3216eb2a..839e4c773f34aba785deb46f8e6b12171644cda6 100644
--- a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.html
+++ b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.html
@@ -218,12 +218,12 @@ div.tocify {
<h1 class="title toc-ignore">The Changing Nature of Peak Demand in the UK: 1974 - 2014</h1>
<h4 class="author"><em>Ben Anderson (<a href="mailto:b.anderson@soton.ac.uk">b.anderson@soton.ac.uk</a>, <code>@dataknut</code>), Jacopo Torriti (<a href="mailto:j.torriti@reading.ac.uk">j.torriti@reading.ac.uk</a>, <code>@JTorriti</code>)</em></h4>
-<h4 class="date"><em>Last run: 2018-01-22 23:25:21</em></h4>
+<h4 class="date"><em>Last run: 2018-03-08 04:01:31</em></h4>
</div>
-<pre><code>## [1] "Loading functions from /Users/ben/github/uosSERG/DEMAND/demandFunctions.R"</code></pre>
+<pre><code>## [1] "Loading functions from /Users/ben/gitlabSoton/SERG/DEMAND/demandFunctions.R"</code></pre>
<pre><code>## [1] "Loading the following libraries using lb_myRequiredPackages: data.table"
## [2] "Loading the following libraries using lb_myRequiredPackages: ggplot2"
## [3] "Loading the following libraries using lb_myRequiredPackages: readr"
@@ -260,6 +260,14 @@ div.tocify {
<li>Most models (MARKAL, TIMES) tend to quantify change of energy demand in the future as average consumption per day/per year (exeption from EST)</li>
<li>Areas in which time use data has been used longitudinally (not in energy)</li>
</ul>
+<div id="national-grid-system-peaks" class="section level2">
+<h2><span class="header-section-number">3.1</span> National Grid system peaks</h2>
+<p>We use this data to try to examine changing peaks over time. Note that these demand values include <em>all</em> demand, not just households.</p>
+<p><img src="changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load%20NG%20demand%20data-1.png" /><!-- --><img src="changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load%20NG%20demand%20data-2.png" /><!-- --><img src="changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load%20NG%20demand%20data-3.png" /><!-- --></p>
+<p>The last chart shows a quite substantial drop in demand over the last 10 years… at all times of year.</p>
+<p>The next two charts show demand levels in January 2006 & 2016 as mean MW per hour.</p>
+<p><img src="changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compareNGJanuary2006toJanuary2016-1.png" /><!-- --><img src="changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compareNGJanuary2006toJanuary2016-2.png" /><!-- --></p>
+</div>
</div>
<div id="methods" class="section level1">
<h1><span class="header-section-number">4</span> Methods</h1>
@@ -278,6 +286,7 @@ div.tocify {
<ul>
<li><a href="(http://www.timeuse.org/mtus)">MTUS World 6</a> - Multinational Timeuse Survey sample for the UK 1974-2005. Note that the most recent MTUS release is W9 but as far as we can tell it has no additional data of use in this paper;</li>
<li><a href="https://discover.ukdataservice.ac.uk/catalogue/?sn=8128">UK Time Use Survey 2014-2015</a> data converted to MTUS format.</li>
+<li><a href="https://www.nationalgrid.com/uk/electricity/market-operations-and-data/data-explorer">National Grid 2005-2016 England & Wales demand data</a></li>
</ul>
<p>The following section loads the <a href="https://dataknut.github.io/UK-TU-2014/convertToMTUS/createMTUSFromUKTU2014.html">harmonised</a> MTUS & UK TU 2014-2015 data which has then been processed to form a <a href="">synthetic half-hour dataset</a> that can enable comparisons over time. Much of this output will not be necessary for the paper but is useful detail here to understand what we are doing.</p>
<div id="survey-data" class="section level2">
@@ -4685,7 +4694,7 @@ Observations
</div>
<div id="about" class="section level1">
<h1><span class="header-section-number">12</span> About</h1>
-<p>Analysis completed in: 5.817 seconds using <a href="https://cran.r-project.org/package=knitr">knitr</a> with R version 3.4.2 (2017-09-28) running on x86_64-apple-darwin15.6.0.</p>
+<p>Analysis completed in: 7.095 seconds using <a href="https://cran.r-project.org/package=knitr">knitr</a> with R version 3.4.2 (2017-09-28) running on x86_64-apple-darwin15.6.0.</p>
<p>R packages used:</p>
<ul>
<li>base R - for the basics <span class="citation">[@baseR]</span></li>
diff --git a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.md b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.md
index dc6e5117cad763f231219e28b510f40a19c64dab..29e6a97e977b109f91d170da3c46435fd28fcbdf 100644
--- a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.md
+++ b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0.md
@@ -1,7 +1,7 @@
---
title: "The Changing Nature of Peak Demand in the UK: 1974 - 2014"
author: "Ben Anderson (b.anderson@soton.ac.uk, `@dataknut`), Jacopo Torriti (j.torriti@reading.ac.uk, `@JTorriti`)"
-date: 'Last run: 2018-01-22 23:25:21'
+date: 'Last run: 2018-03-08 04:01:31'
output:
html_document:
fig_caption: yes
@@ -24,7 +24,7 @@ output:
```
-## [1] "Loading functions from /Users/ben/github/uosSERG/DEMAND/demandFunctions.R"
+## [1] "Loading functions from /Users/ben/gitlabSoton/SERG/DEMAND/demandFunctions.R"
```
@@ -63,6 +63,19 @@ output:
* Most models (MARKAL, TIMES) tend to quantify change of energy demand in the future as average consumption per day/per year (exeption from EST)
* Areas in which time use data has been used longitudinally (not in energy)
+## National Grid system peaks
+
+We use this data to try to examine changing peaks over time. Note that these demand values include _all_ demand, not just households.
+
+<!-- --><!-- --><!-- -->
+
+The last chart shows a quite substantial drop in demand over the last 10 years... at all times of year.
+
+The next two charts show demand levels in January 2006 & 2016 as mean MW per hour.
+
+<!-- --><!-- -->
+
+
# Methods
* Details about longitudinal analysis
- NB: this is not `really` longitudinal (following the same individuals over time), it is analysis of a series of cross-sectional samples
@@ -75,6 +88,7 @@ Data used:
* [MTUS World 6]((http://www.timeuse.org/mtus)) - Multinational Timeuse Survey sample for the UK 1974-2005. Note that the most recent MTUS release is W9 but as far as we can tell it has no additional data of use in this paper;
* [UK Time Use Survey 2014-2015](https://discover.ukdataservice.ac.uk/catalogue/?sn=8128) data converted to MTUS format.
+ * [National Grid 2005-2016 England & Wales demand data](https://www.nationalgrid.com/uk/electricity/market-operations-and-data/data-explorer)
The following section loads the [harmonised](https://dataknut.github.io/UK-TU-2014/convertToMTUS/createMTUSFromUKTU2014.html) MTUS & UK TU 2014-2015 data which has then been processed to form a [synthetic half-hour dataset]() that can enable comparisons over time. Much of this output will not be necessary for the paper but is useful detail here to understand what we are doing.
@@ -137,6 +151,9 @@ Table: MTUS 1974-2015 Survay data: age ranges (weighted)
2005 0 606 817 926 780 756 492 401 0
2014 1899 2041 2587 2447 2577 2118 1783 1328 0
+
+
+
## Synthetic halfhour MTUS 1974-2014
Now load the synthetic 'half hour' MTUS 1974 - 2014 data we previously created.
@@ -250,6 +267,7 @@ ba_age_r nEpisodes nDiaries nRespondents
66-75 329189 5822 2245
75+ 155258 2730 1180
+
# DEMAND Acts
We use this data to code the main and secondary activities into a non-arbitrary but highly aggregated set of 10 DEMAND 'Acts'. This enables us to more easily depict change over time at a coarse level. The aggregated codes are as follows:
@@ -1112,7 +1130,7 @@ If you wish to cite this work please use:
# About
-Analysis completed in: 5.817 seconds using [knitr](https://cran.r-project.org/package=knitr) with R version 3.4.2 (2017-09-28) running on x86_64-apple-darwin15.6.0.
+Analysis completed in: 7.095 seconds using [knitr](https://cran.r-project.org/package=knitr) with R version 3.4.2 (2017-09-28) running on x86_64-apple-darwin15.6.0.
R packages used:
diff --git a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compare January 2006 & January 2016-1.png b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compare January 2006 & January 2016-1.png
new file mode 100644
index 0000000000000000000000000000000000000000..3172f9b7f96db0d184deae69d7372843dbc0931c
Binary files /dev/null and b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compare January 2006 & January 2016-1.png differ
diff --git a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compare January 2006 & January 2016-2.png b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compare January 2006 & January 2016-2.png
new file mode 100644
index 0000000000000000000000000000000000000000..e6cae76d794936110ecaf6ece39d3ea054e95745
Binary files /dev/null and b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compare January 2006 & January 2016-2.png differ
diff --git a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compareNGJanuary2006toJanuary2016-1.png b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compareNGJanuary2006toJanuary2016-1.png
new file mode 100644
index 0000000000000000000000000000000000000000..994f6a6cef082674c798d835c43e396987de6c70
Binary files /dev/null and b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compareNGJanuary2006toJanuary2016-1.png differ
diff --git a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compareNGJanuary2006toJanuary2016-2.png b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compareNGJanuary2006toJanuary2016-2.png
new file mode 100644
index 0000000000000000000000000000000000000000..4d412401671c81698ab083e2eb81be70e5fe723a
Binary files /dev/null and b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/compareNGJanuary2006toJanuary2016-2.png differ
diff --git a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load NG demand data-1.png b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load NG demand data-1.png
new file mode 100644
index 0000000000000000000000000000000000000000..b9cd7d4e80a008c4dfe69fb5b46e9673c717799e
Binary files /dev/null and b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load NG demand data-1.png differ
diff --git a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load NG demand data-2.png b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load NG demand data-2.png
new file mode 100644
index 0000000000000000000000000000000000000000..426000ff5af544328154d60730403fa29a84505e
Binary files /dev/null and b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load NG demand data-2.png differ
diff --git a/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load NG demand data-3.png b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load NG demand data-3.png
new file mode 100644
index 0000000000000000000000000000000000000000..a6c8925a0c8320007e1c38ebf591d1f110af8b27
Binary files /dev/null and b/Theme-1/changeOverTime/changingPeakDemandMtus1974_2014_v2.0_files/figure-html/load NG demand data-3.png differ