This directory provides hourly observation files from the non-FEM monitors on AQmap and is updated in near real-time. 

*** ALL DATA ARE EXPLICITLY NOT FOR COMMERCIAL USAGE ***

If these data are used, you must cite AQmap AS WELL as the data provider (see the bottom right of https://aqmap.ca for data attributions). 

Dates are UTC time, *forward-looking* hourly averages, meaning a date of "2019-01-01 00:00" corresponds to the period of "2019-01-01 00:00" to "2019-01-01 00:59"

A list of files in each monitoring network's directory can be found within respective "file_list_{network}.txt" files in this directory. Files follow this naming convention:
./{network}/{year}/{month}/{observationDate}_aq.txt:AQ:{network}:AQCSV:{fileCreationDate}

Each file has the following columns:
*pm10_raw is also provided for liveable cities monitors
*no bias correction is currently applied to the liveable cities data

1. site - Monitoring site identifier. Begins with "124" for sites in Canada or "840" for sites in the US, followed by the monitor's network-specific site ID. "./aqcsv_stations.csv" is provided for retrieving additional site information.
2. lat - Latitude of site in decimal degrees (datum = WGS84)
3. lon - Longitude of site in decimal degrees (datum = WGS84)
4. datetime - Forward-looking, UTC datestamp of observation in this format "{year}{month}{day}T{hour}{minute}+0000"
5. pm25_raw - Hourly mean fine particulate matter (PM2.5) concentration (ug/m3) directly from the monitor (NOT bias corrected)
6. pm25_corrected - Hourly mean fine particulate matter (PM2.5) concentration (ug/m3) with the UNBC/ECCC bias correction applied (if applicable)
7. pm25_a_corrected - Hourly mean fine particulate matter (PM2.5) concentration (ug/m3) from the "A" sensor (if available) with the UNBC/ECCC bias correction applied (if applicable)
8. pm25_b_corrected - Hourly mean fine particulate matter (PM2.5) concentration (ug/m3) from the "B" sensor (if available) with the UNBC/ECCC bias correction applied (if applicable)
9. pm25_FEM - Hourly mean fine particulate matter (PM2.5) concentration (ug/m3) from the nearest regulatory (FEM) monitor
10. nearest_FEM - The station ID for the nearest FEM monitor to this site
11. distance_FEM - The distance in km between the nearest FEM and this site
12. temperature - Hourly mean temperature (degC) from the site (may be influenced by internal sensor heat and incoming solar radiation)
13. humidity - Hourly mean relative humidity (%) from the site (may be influenced by internal sensor heat and incoming solar radiation)
14. pressure - Hourly mean air pressure from the site (if available)

The following R code can be used to load AQCSVs for (a) particular network(s)/date(s):

require(dplyr) # for data wrangling
require(data.table) # for quickly downloading/importing data
require(stringr) # for working with text
require(lubridate) # for working with dates

# Function to get AQCSV paths from the file_list_{network}.txt files
# dates can either be "all" (not recommended due to large number of files)
# or a vector of datetimes (forward-looking!)
make_aqcsv_path = function(dates = "all", network = "PA") {
  # Allowed networks and their respective AQCSV directories
  networks = list(
    "PA"  = "purpleair", 
    "EGG" = "aqegg", 
    "LC"  = "liveablecities")
  # Stop if invalid network
  network = toupper(network)
  if(! network %in% names(networks)) stop("Invalid network")

  # URL to AQCSV directory
  base_url = "https://aqmap.ca/aqmap/data/AQCSV/"
  
  # Function to parse file paths from file_list_{network}.txt
  parse_aqcsv_file_list = function(aqcsvs_dir){
    file_list = str_remove_all(aqcsvs_dir, pattern = "/\\{year\\}|/\\{month\\}|/unvalidated") %>%
      stringr::str_replace("AQCSV/", "AQCSV/file_list_") %>%
      paste0(".txt")
    file_list %>%
      fread(header = F) %>%
      unlist() %>% unname() %>%
      file.path(dirname(file_list), .) %>%
      setNames(., str_extract(., "\\d{8}_\\d{4}") %>%
                 lubridate::ymd_hm())
  }
  
  # Get all AQCSV files for desired network
  aqcsv_files = paste0(base_url, networks[[network]]) %>%
    parse_aqcsv_file_list() 
  
  # Filter to desired dates if not all dates desired
  if(!"all" %in% dates) aqcsv_files = aqcsv_files %>%
    {.[names(.) %in% format(lubridate::with_tz(dates, "UTC"), "%F %T")]}
  
  return(aqcsv_files)
}

# Function to download, import, and format an AQCSV file
read_aqcsv = function(aqcsv_path, use_sf = FALSE){
  # Download/import file 
  aqcsv = data.table::fread(aqcsv_path, data.table = FALSE, 
                    colClasses = "character") %>% 
    # Convert numeric columns from text to numbers
    mutate(across(-c("site", "datetime"), as.numeric)) %>%
    # Convert datetime from text to a datetime object
    mutate(datetime = datetime %>% 
             str_remove_all("\\+0000$|T") %>% 
             ymd_hm(tz = "UTC"),
	   # And the site identifier to a factor from text
           site = factor(site))
  # If use_sf == TRUE, convert to a spatial point object using the sf package
  if(use_sf){
    aqcsv = sf::st_as_sf(aqcsv, coords = c("lon", "lat"))
  }
  
  return(aqcsv)
}

# Specify desired network
network = "PA" #or "EGG" or "LC"
# Specify desired dates (hourly forward-looking)
dates = seq(
  ymd_h("2019-01-01 00", tz = "UTC"), 
  ymd_h("2019-01-31 23", tz = "UTC"), 
  by = "1 hours"
)
# Get aqcsv paths for that network and those dates
aqcsvs = make_aqcsv_path(dates, network) %>%
  # Load each of those files into a list of data frames
  lapply(read_aqcsv) %>%
  # Bind list into a single data frame
  bind_rows()