analyse-NEED-EULF-2014-descriptives.do

* Script to analyse DECC's NEED data to:
* examine distributions etc

* NB this script uses 2 data files derived from the original data using the 'process' script

* Original data available from: UK DATA ARCHIVE: Study Number 7518 - National Energy Efficiency Data-Framework, 2014
* http://discover.ukdataservice.ac.uk/catalogue/?sn=7518

* Ben Anderson, Energy & Climate Change, Faculty of Engineering & Environment, University of Southampton
* b.anderson@soton.ac.uk
* (c) University of Southampton

* Unless there is a different license file in the folder in which this script is found, the Creative Commons Attribution-NonCommercial 4.0 International (CC BY-NC 4.0) license applies
* http://creativecommons.org/licenses/by-nc/4.0/

clear all

capture noisily log close

* written for Mac OSX - remember to change filesystem delimiter for other platforms
local home "/Users/ben/Documents"
local proot "`home'/Work/Data/Social Science Datatsets/DECC"
* for clam
* local proot "`home'/Work/NEED"
local dpath "`proot'/NEED/End User Licence File 2014/processed"
local rpath "`proot'/results/NEED"

local version "v1.1"

* quick tests for 2012
local do_2012_desc = 1

* tests for all years using long file - takes a while
local do_long_desc = 1 

* toggle graph drawing
local do_graphs = 0

set more off

log using "`rpath'/analyse-NEED-EULF-2014-descriptives-`version'-$S_DATE.smcl", replace

if `do_2012_desc' {
	* use a subsample for speed
	local sample = "20pc"
	* first use the wide file for basic descrpitives
	use "`dpath'/need_eul_may2014_consumptionfile_wide_`sample'.dta", clear
	* match in the xwave file with the vars we want
	merge 1:1 HH_ID using "`dpath'/need_eul_may2014_xwavefile_`sample'", keepusing(EE_BAND FLOOR_AREA_BAND PROP_AGE)
	* distributions for 2012 (to test)
	* processor intensive
	local vars "Econs2012 Gcons2012"
	local tvars "EE_BAND FLOOR_AREA_BAND PROP_AGE"
	* test values for valid - check for valid 0s for example. This only happens for gas where:
	* 100 < gcons < 250 so included but rounded to nearest 500 = 0
	
	* elec always rounded to nearest 50 so min shoudl always be 100

	foreach v of local vars {
		tabstat `v', by(`v'Valid) s(n mean semean min max)
		foreach tv of local tvars {
			di "***************"
			di "* Testing `tv'"
			* test values for `tv' - check for 0s for example
			tabstat `v' if `v'Valid == "V", by(`tv') s(n mean semean min max)
			tab `v' if `v' < 1000
			if `do_graphs' {
				histogram `v' if `v'Valid == "V", by(`tv') name(h_`tv'_`v'_`sample')
				graph export "`rpath'/NEED-EULF-2014-histo_`v'_by_`tv'_`sample'_valid.png", replace 
			
				graph box `v' if `v'Valid == "V", over(`tv') name(b_`tv'_`v'_`sample')
				graph export "`rpath'/NEED-EULF-2014-box_`v'_by_`tv'_`sample'_valid.png", replace 
			}
		}
	}
}

if `do_long_desc' {
	* Now use the pre-processed long form file which contains all years of consumption data but not the constant values (housing charactersitics etc) which are in the xwave file
	* do this for each random sample of differening sizes as a check
	local samples "10 20 30 40 50 100"
	foreach s of local samples {
		di "************************"
		di "* Using `s'% sample"
		use "`dpath'/need_eul_may2014_consumptionfile_long_`s'pc.dta", clear
	
		* set as panel in case it wasn't
		xtset HH_ID year

		* examine panel status
		xtdescribe 
		
		* distributions for valid obs
		local vars "Econs Gcons"
		foreach v of local vars {
			* overall
			xtsum `v' if `v'Valid == "V"
			* test values for valid - check for valid 0s for example. This only happens for gas where:
			* 100 < gcons < 250 so included but rounded to nearest 500 = 0
	
			* elec always rounded to nearest 50 so min shoudl always be 100
			tabstat `v', by(`v'Valid) s(n mean semean min max) 
			* by year
			* check for 0s
			table `v' year if `v' < 1000
			table `v'Valid year, c(count `v' min `v' mean `v' max `v')
			 
			if `do_graphs' {
				histogram `v' if `v'Valid == "V", by(year) name(histo_`s'pc_`v')
				graph export "`rpath'/NEED-EULF-2014-`s'pc-histo_`v'_by_year_valid.png", replace 
				graph box `v' if `v'Valid == "V", over(year) name(box_`s'pc_`v')
				graph export "`rpath'/NEED-EULF-2014-`s'pc-box_`v'_by_year_valid.png", replace 
			}
		}
		* check the years where gas = valid but consumption = 0
		* Presumably in these cases gas > 100 but < 249 ?
	}
}

di "* Done!"

log close