BEGIN {FS=",";oldcat="0";oldType="O";oldsec="O";one=1; misfile="missing_files_"strftime("%Y-%m-%d_%H_%M")".txt" print "Missing files "strftime("%Y-%m-%d %H:%M:S") > misfile # Generate Google Earth KML files for water quality data points. # Generate text HTML files for accessing the same data. # Normally data should be sorted by type, then tertiary|wma, then feature id. # 2007-06-11 Michael Silberbauer major changes from previous versions (icons, dynamic variables in awk) # 2007-08-31 Select better map icons # 2007-09-01 Timespan (see http://nsidc.org/data/virtual_globes/timelines.html) # 2007-09-03 Exclude all boreholes from surface report, not just those with < 10 samples # 2007-09-07 Arrange folders more logically # 2008-05-15 Check if files exist... adds an extra layer of complexity... only works for file-based system # 2008-08-04 Cumulative minor modifications to text for missing data in balloons and HTML tables #Usage # for primary drainage region grouping (WMA_on=0): # gawk -v WMA_on=0 -v bor=0 -v mau=0 -v disk=0 -f C:\data\Program\awk\kml_WMS_mnpts_html.awk C:\Data_Large\av\nms_wms_dat_pri.txt # 84960,'A1H002','A1H002 DINOKANA AT DAM','',-25.448056,25.878889,'A10',0,0,'1899-12-30','1899-12-30','Dinokana at Dam','A10_0000084960',3,'A10A','A1','A' # # for WMA grouping (WMA_on=1): # gawk -v WMA_on=1 -v bor=0 -v mau=0 -v disk=0 -f C:\data\Program\awk\kml_WMS_mnpts_html.awk C:\Data_Large\av\nms_wms_dat_wma.txt # 90174,'A2H027Q01','A2H027Q01 PIENAARS RIVER AT BAVIAANSPOORT','Rivers',-25.662500,28.351389,'A23',1,4287,'1967-02-08','2008-02-25','Pienaars River at Baviaanspoort','A23_0000090174',3,'A23A','A2','A' # # borehole and WMA switches are set in C:\data\Program\awk\kml_WMS.bat using -v variable setting for gawk # Choose boreholes or no boreholes (many of the sites are boreholes with one sample, which clutter the view): #bor=0# use batch file with: gawk -v bor=0 -v WMA_on=0 ... if(bor)borf="_boreh" else borf="_nobor" if(bor)bfold=" boreholes" else bfold=" surface" # switch WMA mode off (0) or on (1, only for data sorted by WMA, then tertiary, then type, then feature id): #WMA_on=0# use batch file with: gawk -v bor=1 -v WMA_on=1 ... # choose whether to use Maucha symbols as point markers (use -v mauc=1): if(mau) msym=1 else msym=0 if(mau) bfold=bfold" Maucha" # set the disk to the local disk (for testing) or "" for online (now set in calling routine): #disk="" #disk="c:" # set the address root (hrfc="file:///" or "http://") hrfc="file:///" isdisk=1 # set the address path (dpath="/data_large/wms_graphs_2006/" or "www.dwaf.gov.za/iwqs/wms/data/") dpath="/data_large/wms_graphs_2006/" if (disk==""||disk==0) { hrfc="http://" disk="" isdisk=0 dpath="www.dwaf.gov.za/iwqs/wms/data/" } #print "Disk="disk", isdisk="isdisk dospath=dpath gsub("/","\\",dospath) if (isdisk) dpath="www.dwaf.gov.za/iwqs/wms/data/" # set the search path to check if files exist, when creating the web version srchpath="Q:\\wms\\data\\" #srchpath="C:\\data_large\\wms_graphs_2008\\" # set Water Management Area numbers and names: WMA[00]="sites outside borders" WMA[01]="Limpopo" WMA[02]="Luvuvhu and Letaba" WMA[03]="Crocodile West and Marico" WMA[04]="Olifants" WMA[05]="Inkomati" WMA[06]="Usutu to Mhlatuze" WMA[07]="Thukela" WMA[08]="Upper Vaal" WMA[09]="Middle Vaal" WMA[10]="Lower Vaal" WMA[11]="Mvoti to Umzimkulu" WMA[12]="Mzimvubu to Keiskamma" WMA[13]="Upper Orange" WMA[14]="Lower Orange" WMA[15]="Fish to Tsitsikamma" WMA[16]="Gouritz" WMA[17]="Olifants-Doorn" WMA[18]="Breede" WMA[19]="Berg" #for(i=1;i<=19;i++) print i,WMA[i] # set marker names and files: ico[1]="normalPlacemark" ;nic[1]="http://maps.google.com/mapfiles/kml/paddle/wht-circle-lv.png" ico[2]="unknownPlacemark" ;nic[2]="http://maps.google.com/mapfiles/kml/paddle/wht-blank-lv.png" ico[3]="boreholePlacemark" ;nic[3]="http://maps.google.com/mapfiles/kml/paddle/B-lv.png" ico[4]="borehole00Placemark" ;nic[4]="http://maps.google.com/mapfiles/kml/paddle/O-lv.png" ico[5]="borehole01Placemark" ;nic[5]="http://maps.google.com/mapfiles/kml/paddle/1-lv.png" ico[6]="borehole02Placemark" ;nic[6]="http://maps.google.com/mapfiles/kml/paddle/2-lv.png" ico[7]="borehole03Placemark" ;nic[7]="http://maps.google.com/mapfiles/kml/paddle/3-lv.png" ico[8]="borehole04Placemark" ;nic[8]="http://maps.google.com/mapfiles/kml/paddle/4-lv.png" ico[9]="borehole05Placemark" ;nic[9]="http://maps.google.com/mapfiles/kml/paddle/5-lv.png" ico[10]="borehole06Placemark" ;nic[10]="http://maps.google.com/mapfiles/kml/paddle/6-lv.png" ico[11]="borehole07Placemark" ;nic[11]="http://maps.google.com/mapfiles/kml/paddle/7-lv.png" ico[12]="borehole08Placemark" ;nic[12]="http://maps.google.com/mapfiles/kml/paddle/8-lv.png" ico[13]="borehole09Placemark" ;nic[13]="http://maps.google.com/mapfiles/kml/paddle/9-lv.png" ico[14]="borehole10Placemark" ;nic[14]="http://maps.google.com/mapfiles/kml/paddle/10-lv.png" ico[15]="canalPlacemark" ;nic[15]="http://maps.google.com/mapfiles/kml/pal5/icon50l.png" ico[16]="potablePlacemark" ;nic[16]="http://maps.google.com/mapfiles/kml/paddle/blu-blank-lv.png" ico[17]="industryPlacemark" ;nic[17]="http://maps.google.com/mapfiles/kml/pal2/icon10.png" #ico[18]="hydroPlacemark" ;nic[18]="http://maps.google.com/mapfiles/kml/shapes/water.png" ico[18]="hydroPlacemark" ;nic[18]="http://maps.google.com/mapfiles/kml/paddle/blu-circle-lv.png" #ico[19]="lakePlacemark" ;nic[19]="http://www.dwaf.gov.za/iwqs/wms/img/dam.png" ico[19]="lakePlacemark" ;nic[19]="http://maps.google.com/mapfiles/kml/paddle/ltblu-blank-lv.png" ico[20]="marinePlacemark" ;nic[20]="http://maps.google.com/mapfiles/kml/paddle/grn-stars-lv.png" ico[21]="municipalPlacemark" ;nic[21]="http://maps.google.com/mapfiles/kml/pal3/icon21.png" ico[22]="pipelinePlacemark" ;nic[22]="http://maps.google.com/mapfiles/kml/shapes/target.png" ico[23]="sewagePlacemark" ;nic[23]="http://maps.google.com/mapfiles/kml/paddle/ylw-blank.png" #ico[24]="tunnelPlacemark" ;nic[24]="http://maps.google.com/mapfiles/kml/pal4/icon25.png" ico[24]="tunnelPlacemark" ;nic[24]="http://maps.google.com/mapfiles/kml/shapes/donut.png" ico[25]="wetlandPlacemark" ;nic[25]="http://maps.google.com/mapfiles/kml/paddle/grn-blank-lv.png" ico[26]="groundPlacemark" ;nic[26]="http://maps.google.com/mapfiles/kml/paddle/g-lv.png" ico[27]="panPlacemark" ;nic[27]="http://maps.google.com/mapfiles/kml/paddle/ylw-blank-lv.png" } { gsub( "&", " and ", $0 ) #gsub( "'", "", $0 ) # single quotes are an artefact of the ArcInfo Tables UNLOAD command in wms2nms.aml #for( i=1;i<17;i++ ) print i,$i for( i=1;i<17;i++ ) gsub( "'", "", $i) # single quotes are an artefact of the ArcInfo Tables UNLOAD command in wms2nms.aml ptnwms=$1 # Monitoring_point_id ptstn =$2 #station code pttype=$3 # Located_on_type pntlat=$4 # Latitude pntlon=$5 # Longitude catcht=$6 # Tertiary ... quat is $15 ptactv=$7 # Active point flag nsmpls=$8 # Number_of_samples dtsmp1=$9 # First_sample_date dtsmpn=$10 # Last_sample_date ptname=$11 # Monitoring_point_name pntsrt=$12 # PRI_Sort (tertiary plus WMS code, stuffed with zeros e.g. D16_0000157611 wmanum=$13 # WMA number qat =$14 # Quaternary drainage region sec =$15 # Secondary drainage region primry=$16 # Primary drainage region or WMA number ... now just primary catcht=substr( qat,1,3 ) # Warning: the WMS and GIS tertiaries differ for some sites! cat=substr( catcht,1,1 ) wma="WMA"wmanum if(wmanum<10) wma="WMA0"wmanum #insert leading zero to tidy up file name #if(wmanum>=10 && wmanum<100) # here's a switch to group by WMAs if(WMA_on) cat=wma ter=tolower(substr(catcht,1,3)) sec=tolower(substr(catcht,1,2)) pri=tolower(substr(catcht,1,1)) #print "Catchment "cat if( cat!=oldcat ) { if (one) { one=0 # the first time here we skip the lines to close a file } else { # we've come to the end of a section print " " >> catfile # for secondary catchment print "" >> catfile # for Type print "" >> catfile # for primary catchment or WMA print "" >> catfile close (catfile) #print "Closing "oldcat " and opening "cat print "" >> htmfile print ""strftime("%Y-%m-%d %H:%M") >> htmfile print "|WMS water quality sites home" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile close (htmfile) } oldcat=cat oldType="O" oldsec="O" cathead=toupper(cat)"_reg_WMS"borf htmfile=cathead".htm" #the HTML files for maucha are the same as for non-Maucha -only the KML differs if(msym)cathead=cathead"_m" # catfold=toupper(cat)" region"bfold catfold=toupper(cat)" "bfold catfile=cathead".kml" secfold=toupper(sec) print strftime("%Y-%m-%d %H:%M:%S")" "catfile,NR"+" # write the header for this grouping (primary catchment or WMA) print "" > htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile if(WMA_on) print WMA[wmanum]" WMA "strftime("%Y-%m-%d %H:%M") borf", RQS WMS water quality data Department of Water Affairs and Forestry" >> htmfile else print toupper(cat)" region " strftime("%Y-%m-%d %H:%M") borf", RQS WMS water quality data Department of Water Affairs and Forestry" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile if(WMA_on) print "" >> htmfile else print "" >> htmfile print "" >> htmfile print "" >> htmfile print "
\"DWAF

Resource Quality Services water quality data for "WMA[wmanum]" WMA.

Resource Quality Services water quality data for region "toupper(cat)".

\"WMS
" >> htmfile print "
" >> htmfile print "WMS water quality sites home" >> htmfile print "
" >> htmfile print "

" >> htmfile if(bor) print "List of boreholes only" >> htmfile else print "List of sites (other than boreholes)" >> htmfile if(WMA_on) print "for "WMA[wmanum]" WMA." >> htmfile else print "for region "toupper(cat)"." >> htmfile print "

" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile print "" > catfile print "" >> catfile print "" >> catfile # if(WMA_on) print ""WMA[primry]"" >> catfile # else print "WMS inventory "toupper(cat)"" >> catfile if(WMA_on) print ""catfold" "WMA[wmanum]"" >> catfile else print ""catfold"" >> catfile print "" >> catfile print "#wmsPlacemark" >> catfile for (ic in ico) { print "" >> catfile } for (ic in ico) { print "" >> catfile } # markname="S_"toupper(cat)"_SSDDSDDS" print "1" >> catfile # item1="Monitoring_Point_ID" # print "" >> catfile # print "" >> catfile # print "" >> catfile } # this else if comes from way further up... here we decide whether to exclude boreholes and sites with no data: else if( nsmpls>0 && ((pttype=="Borehole" && bor) || (pttype!="Borehole" && !bor)) ) { split(ptname,stn," ") #st=stn[1] st=substr( ptstn,1,9 ) #check for pattern of embedded hydro station code in description e.g. A2R027Q01 # check for pattern of hydro station code in monitoring point name - WMS no longer uses the Q01 suffix consistently his=0 #print cat" "NR": "ptnwms"("st") n="nsmpls if( (st ~ /[A-Z][1-9][A-Z][0-9][0-9][0-9]/ ) || (st ~ /[A-Z][1-9][A-Z][0-9][0-9][0-9][Q][0-9][0-9]/) ) his=1 if( his ) stas=substr(st,1,6) if( his && substr(st,3,1)=="R" ) { stas=substr(st,1,3)1*substr(st,4,3) if( substr(st,7,1)=="Q" ) stas=substr(st,1,3)1*substr(st,4,3)"."1*substr(st,8,2) } if( his && substr(st,3,1)=="H" ) stas=substr(st,1,3)1*substr(st,4,3) if( !his ) { stas=ptnwms if( substr(ptnwms,1,1)>0 && substr(ptnwms,1,1)<=9 && substr(ptnwms,2,4)=="0000" ) # remove some 0s { stas=substr(ptnwms,1,1)"-"1*substr(ptnwms,2,length(ptnwms)) } } ist = "#normalPlacemark" if( pttype == "Class pending") ist = "#unknownPlacemark" if( pttype == "Other Ground Fractures") ist = "#groundPlacemark" if( pttype == "Canal") ist = "#canalPlacemark" if( pttype == "Pipeline") ist = "#pipelinePlacemark" if( pttype == "Pump Station") ist = "#pipelinePlacemark" if( pttype == "Tunnel") ist = "#tunnelPlacemark" if( pttype == "Unknown Transfer Feature Type") ist = "#tunnelPlacemark" if( pttype == "Potable Water Purification Works")ist = "#potablePlacemark" if( pttype == "Excavation - Quarry") ist = "#industryPlacemark" if( pttype == "Industrial Property") ist = "#industryPlacemark" if( pttype == "Mine Property") ist = "#industryPlacemark" if( pttype == "Mine Shaft") ist = "#industryPlacemark" if( pttype == "Mineral Process Plant/Area") ist = "#industryPlacemark" if( pttype == "Municipal Area") ist = "#industryPlacemark" if( pttype == "Slime/Slurry Dam") ist = "#industryPlacemark" if( pttype == "Containment/Emergency/Return Water Dam") ist = "#lakePlacemark" if( pttype == "Dam / Barrage") ist = "#lakePlacemark" if( pttype == "Evaporation Dam") ist = "#lakePlacemark" if( pttype == "Lake") ist = "#lakePlacemark" if( pttype == "Pan") ist = "#panPlacemark" if( pttype == "Reservoir") ist = "#lakePlacemark" if( pttype == "Spring/Eye") ist = "#groundPlacemark" if( pttype == "Wetland") ist = "#wetlandPlacemark" if( pttype == "Estuary/Lagoon") ist = "#marinePlacemark" if( pttype == "Ocean / Bay") ist = "#marinePlacemark" if( pttype == "Monitoring Points") ist = "#hydroPlacemark" if( pttype == "Rivers") ist = "#hydroPlacemark" if( pttype == "Sewage Works") ist = "#sewagePlacemark" if( pttype == "Solid Waste Site") ist = "#sewagePlacemark" if( pttype == "Storm Water System") ist = "#sewagePlacemark" if( pttype == "Water/Effluent Treatment Plant") ist = "#sewagePlacemark" istl=ist if( pttype == "Borehole") { ist = "#boreholePlacemark" istl = "#boreholePlacemark" if(nsmpls>0 && nsmpls<10) ist = "#borehole0"nsmpls"Placemark" if( nsmpls == 10 ) ist = "#borehole10Placemark" if( nsmpls >= 10 ) ist = "#boreholePlacemark" } if(pttype!=oldType) { if(oldType!="O") print "" >> catfile print "" >> catfile print " "pttype"" >> catfile print " "istl"L" >> catfile print " 0" >> catfile oldType=pttype oldsec="O" } if(sec!=oldsec) { if(oldsec!="O") print "" >> catfile print "" >> catfile print " "toupper(sec)"" >> catfile print " 0" >> catfile oldsec=sec } hstm = srchpath""ter"/"ter"_"ptnwms fstm = hrfc""disk""dpath""ter"/"ter"_"ptnwms hpdf = hstm".pdf" fpdf = fstm".pdf" hjpg = hstm".jpg" fjpg = fstm".jpg" hzip = hstm".zip" fzip = fstm".zip" #print "hpdf="hpdf;print "fpdf="fpdf # print "<"markname">" >> catfile print "" >> catfile print ""stas"" >> catfile print ""ptname"" >> catfile print "> catfile if(!isdisk&&exists(hjpg) ) print "" >> catfile if( isdisk&&exists(fjpg) ) print "" >> catfile print "WMS "toupper(catcht)"_"ptnwms"
" >> catfile #if(his) print substr(st,1,6)"
" >> catfile print ptstn "
" >> catfile print ptname"
" >> catfile print ""pttype" samples: "nsmpls"
" >> catfile if(nsmpls==1) print dtsmp1"
" >> catfile else print dtsmp1" to "dtsmpn"
" >> catfile #Exclude latitude and longitude: superfluous information in a spatial viewer... if(!isdisk&&exists(hpdf) ) print "graph|" >> catfile if(!isdisk&&!exists(hpdf)) { print "No "fpdf >> misfile print "data not found|" >> catfile } if( isdisk&&exists(fpdf) ) print "graph|" >> catfile # linking directly to a ZIP file seems to be forbidden in Google Earth: #print "data|" >> catfile # so instead link to the station record in the region's HTML page: if(!isdisk&&exists(hzip) ) print "data|" >> catfile if( isdisk&&exists(fzip) ) print "data|" >> catfile if(his) print "flow|" >> catfile print "home|" >> catfile if(!isdisk&&exists(hjpg) ) print "Maucha key|" >> catfile if( isdisk&&exists(fjpg) ) print "Maucha key|" >> catfile #print "" >> catfile #print "report error" >> catfile print "]]>
" >> catfile print "
" >> htmfile print "" >> htmfile if(his) print "" >> htmfile else print "" >> htmfile print "" >> htmfile if(msym) { print "" >> catfile print "#"stas"" >> catfile } else print ""ist"" >> catfile if(nsmpls==1) { print "" >> catfile print ""dtsmp1"" >> catfile print "" >> catfile } else { print "" >> catfile print " "dtsmp1"" >> catfile print " "dtsmpn"" >> catfile print "" >> catfile } print "" >> catfile print " "pntlon","pntlat",0" >> catfile print "" >> catfile # print "<"item1">"ptnwms"" >> catfile print "" >> catfile # print "" >> catfile } } function exists(file, dummy, ret) { # Tim Menzies , tim@menzies.us, http://menzies.us ret=0; if ( (getline dummy < file) >=0 ) {ret = 1; close(file)}; return ret; } END { print " " >> catfile print "" >> catfile print "" >> catfile print "" >> catfile close (catfile) #print "
Water qualityDescriptionTypenFirst
date
Last
date
Flow,
if any
LatLon
" >> htmfile print ""toupper(catcht)"_"ptnwms"|" >> htmfile if(!isdisk&&exists(hpdf) ) print "graph|" >> htmfile if(!isdisk&&!exists(hpdf)) print "data not found|" >> htmfile if( isdisk&&exists(fpdf) ) print "graph|" >> htmfile if(!isdisk&&exists(hzip) ) print "data|" >> htmfile if( isdisk&&exists(fzip) ) print "data|" >> htmfile print ""ptname""pttype""nsmpls""dtsmp1""dtsmpn""substr(st,1,6)"n/a" >> htmfile printf "%10.5f",1*pntlat >> htmfile print "" >> htmfile printf "%9.5f",1*pntlon >> htmfile print "
" >> htmf #print ""strftime("%Y-%m-%d %H:%M") >> htmf #print "" >> htmf #print "" >> htmf #close (htmf) print "" >> htmfile print ""strftime("%Y-%m-%d %H:%M") >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile print "" >> htmfile close (htmfile) }