fixed processing script to give random samples

set the % you want using the ‘sample’  local. By default this
constrains the selection to keep the overall ratios of EE_BAND
PROP_TYPE constant (as in the original sampling)

NEED/process-NEED-EULF-2014.do

@@ -88,10 +88,13 @@ if `create_xwavefile' {
 	* create a wide consumption file
 	preserve
 		keep HH_ID Gcons* Econs*
+		compress
 		save "`dpath'/processed/`ifile'_consumptionfile_wide_`samplet'.dta", replace
 	restore
-	drop Gcons* Econs*
 	
+	
+	preserve
+		drop Gcons* Econs*
 		* fix some mis-codings (or lack of coding of missing)
 		local vars "E7Flag2012 CWI LI"
 		foreach v of local vars {
@@ -140,9 +143,11 @@ if `create_xwavefile' {
 		di "** no idea what G, L, M mean in the 'valid' variables - presumably 0 = off gas & V = valid?"
 		di "** urban/rural would be helpful"
 		codebook
+		compress
 		save "`dpath'/processed/`ifile'_xwavefile_`samplet'.dta", replace
 		log close cb_xwave
 		log on main
+	restore
 }
 
 if `create_longfile' {
@@ -169,11 +174,16 @@ if `create_longfile' {
 	rename _j year
 	* set as panel
 	xtset HH_ID year
+	di "* check distributions for `samplet' sample"
+	xtdescribe
+	xtsum Gcons Econs
 	compress
 	save "`dpath'/processed/`ifile'_consumptionfile_long_`samplet'.dta", replace
+	* this leaves us with the long form file in memory
 }
 
 /*
+* Link xwave data to long form file
 * THIS TAKES AGES and creates a 1.5 GB file for the full dataset - use with care
 * now just merge them
 * start with long file which may or may not have just been re-created
@@ -184,4 +194,6 @@ merge m:1 HH_ID using "`dpath'/`dfile'_xwavefile.dta"
 save "`dpath'/`dfile'_consumptionfile_long_complete_`samplet'.dta", replace
 */
 
+* done!
+
 log close _all