Merge branch 'measurements_archive' into development
commit
de3a05bf97
@ -0,0 +1,135 @@
|
||||
# client for archive.opensensemap.org
|
||||
# in this archive, CSV files for measurements of each sensor per day is provided.
|
||||
|
||||
#' Returns the default endpoint for the archive *download*
|
||||
#' While the front end domain is archive.opensensemap.org, file downloads
|
||||
#' are provided via sciebo.
|
||||
osem_archive_endpoint = function () {
|
||||
'https://uni-muenster.sciebo.de/index.php/s/HyTbguBP4EkqBcp/download?path=/data'
|
||||
}
|
||||
|
||||
#' Fetch day-wise measurements for a single box from the openSenseMap archive.
|
||||
#'
|
||||
#' This function is significantly faster than \code{\link{osem_measurements}} for large
|
||||
#' time-frames, as daily CSV dumps for each sensor from
|
||||
#' \href{http://archive.opensensemap.org}{archive.opensensemap.org} are used.
|
||||
#' Note that the latest data available is from the previous day.
|
||||
#'
|
||||
#' By default, data for all sensors of a box is fetched, but you can select a
|
||||
#' subset with a \code{\link[dplyr]{dplyr}}-style NSE filter expression.
|
||||
#'
|
||||
#' The function will warn when no data is available in the selected period,
|
||||
#' but continue the remaining download.
|
||||
#'
|
||||
#' @param x A `sensebox data.frame` of a single box, as retrieved via \code{\link{osem_box}},
|
||||
#' to download measurements for.
|
||||
#' @param fromDate Start date for measurement download.
|
||||
#' @param toDate End date for measurement download (inclusive).
|
||||
#' @param sensorFilter A NSE formula matching to \code{x$sensors}, selecting a subset of sensors.
|
||||
#' @param progress Whether to print download progress information, defaults to \code{TRUE}.
|
||||
#' @return A \code{tbl_df} Containing observations of all selected sensors for each time stamp.
|
||||
#'
|
||||
#' @seealso \href{https://archive.opensensemap.org}{openSenseMap archive}
|
||||
#' @seealso \code{\link{osem_measurements}}
|
||||
#' @seealso \code{\link{osem_box}}
|
||||
#'
|
||||
#' @export
|
||||
osem_measurements_archive = function (x, ...) UseMethod('osem_measurements_archive')
|
||||
|
||||
#' @export
|
||||
osem_measurements_archive.default = function (x, ...) {
|
||||
# NOTE: to implement for a different class:
|
||||
# in order to call `archive_fetch_measurements()`, `box` must be a dataframe
|
||||
# with a single row and the columns `X_id` and `name`
|
||||
stop(paste('not implemented for class', toString(class(x))))
|
||||
}
|
||||
|
||||
#' @describeIn osem_measurements_archive Get daywise measurements for one or
|
||||
#' more sensors of a single box
|
||||
#' @export
|
||||
#' @examples
|
||||
#' # fetch measurements for a single day
|
||||
#' box = osem_box('593bcd656ccf3b0011791f5a')
|
||||
#' m = osem_measurements_archive(box, as.POSIXlt('2018-09-13'))
|
||||
#'
|
||||
#' \donttest{
|
||||
#' # fetch measurements for a date range and selected sensors
|
||||
#' sensors = ~ phenomenon %in% c('Temperatur', 'Beleuchtungsstärke')
|
||||
#' m = osem_measurements_archive(box, as.POSIXlt('2018-09-01'), as.POSIXlt('2018-09-30'), sensorFilter = sensors)
|
||||
#' }
|
||||
osem_measurements_archive.sensebox = function (x, fromDate, toDate = fromDate, sensorFilter = ~ T, progress = T) {
|
||||
if (nrow(x) != 1)
|
||||
stop('this function only works for exactly one senseBox!')
|
||||
|
||||
# filter sensors using NSE, for example: `~ phenomenon == 'Temperatur'`
|
||||
sensors = x$sensors[[1]] %>%
|
||||
dplyr::filter(lazyeval::f_eval(sensorFilter, .))
|
||||
|
||||
# fetch each sensor separately
|
||||
dfs = by(sensors, 1:nrow(sensors), function (sensor) {
|
||||
df = archive_fetch_measurements(x, sensor$id, fromDate, toDate, progress) %>%
|
||||
dplyr::select(createdAt, value) %>%
|
||||
#dplyr::mutate(unit = sensor$unit, sensor = sensor$sensor) %>% # inject sensor metadata
|
||||
dplyr::rename_at(., 'value', function(v) sensor$phenomenon)
|
||||
})
|
||||
|
||||
# merge all data.frames by timestamp
|
||||
dfs %>% purrr::reduce(dplyr::full_join, 'createdAt')
|
||||
}
|
||||
|
||||
#' fetch measurements from archive from a single box, and a single sensor
|
||||
archive_fetch_measurements = function (box, sensor, fromDate, toDate, progress) {
|
||||
dates = list()
|
||||
from = fromDate
|
||||
while (from <= toDate) {
|
||||
dates = append(dates, list(from))
|
||||
from = from + as.difftime(1, units = 'days')
|
||||
}
|
||||
|
||||
http_handle = httr::handle(osem_archive_endpoint()) # reuse the http connection for speed!
|
||||
progress = if (progress && !is_non_interactive()) httr::progress() else NULL
|
||||
|
||||
measurements = lapply(dates, function(date) {
|
||||
url = build_archive_url(date, box, sensor)
|
||||
res = httr::GET(url, progress, handle = http_handle)
|
||||
|
||||
if (httr::http_error(res)) {
|
||||
warning(paste(
|
||||
httr::status_code(res),
|
||||
'on day', format.Date(date, '%F'),
|
||||
'for sensor', sensor
|
||||
))
|
||||
|
||||
if (httr::status_code(res) == 404)
|
||||
return(data.frame(createdAt = character(), value = character()))
|
||||
}
|
||||
|
||||
measurements = httr::content(res, type = 'text', encoding = 'UTF-8') %>%
|
||||
parse_measurement_csv
|
||||
})
|
||||
|
||||
measurements %>% dplyr::bind_rows()
|
||||
}
|
||||
|
||||
#' returns URL to fetch measurements from a sensor for a specific date,
|
||||
#' based on `osem_archive_endpoint()`
|
||||
build_archive_url = function (date, box, sensor) {
|
||||
sensorId = sensor
|
||||
d = format.Date(date, '%F')
|
||||
format = 'csv'
|
||||
|
||||
paste(
|
||||
osem_archive_endpoint(),
|
||||
d,
|
||||
osem_box_to_archivename(box),
|
||||
paste(paste(sensorId, d, sep = '-'), format, sep = '.'),
|
||||
sep = '/'
|
||||
)
|
||||
}
|
||||
|
||||
#' replace chars in box name according to archive script:
|
||||
#' https://github.com/sensebox/osem-archiver/blob/612e14b/helpers.sh#L66
|
||||
osem_box_to_archivename = function (box) {
|
||||
name = gsub('[^A-Za-z0-9._-]', '_', box$name)
|
||||
paste(box$X_id, name, sep='-')
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
% Generated by roxygen2: do not edit by hand
|
||||
% Please edit documentation in R/archive.R
|
||||
\name{build_archive_url}
|
||||
\alias{build_archive_url}
|
||||
\title{returns URL to fetch measurements from a sensor for a specific date,
|
||||
based on `osem_archive_endpoint()`}
|
||||
\usage{
|
||||
build_archive_url(date, box, sensor)
|
||||
}
|
||||
\description{
|
||||
returns URL to fetch measurements from a sensor for a specific date,
|
||||
based on `osem_archive_endpoint()`
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
% Generated by roxygen2: do not edit by hand
|
||||
% Please edit documentation in R/archive.R
|
||||
\name{osem_archive_endpoint}
|
||||
\alias{osem_archive_endpoint}
|
||||
\title{Default endpoint for the archive download
|
||||
front end domain is archive.opensensemap.org, but file download
|
||||
is provided via sciebo}
|
||||
\usage{
|
||||
osem_archive_endpoint()
|
||||
}
|
||||
\description{
|
||||
Default endpoint for the archive download
|
||||
front end domain is archive.opensensemap.org, but file download
|
||||
is provided via sciebo
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
% Generated by roxygen2: do not edit by hand
|
||||
% Please edit documentation in R/archive.R
|
||||
\name{osem_box_to_archivename}
|
||||
\alias{osem_box_to_archivename}
|
||||
\title{replace chars in box name according to archive script:
|
||||
https://github.com/sensebox/osem-archiver/blob/612e14b/helpers.sh#L66}
|
||||
\usage{
|
||||
osem_box_to_archivename(box)
|
||||
}
|
||||
\description{
|
||||
replace chars in box name according to archive script:
|
||||
https://github.com/sensebox/osem-archiver/blob/612e14b/helpers.sh#L66
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
% Generated by roxygen2: do not edit by hand
|
||||
% Please edit documentation in R/archive.R
|
||||
\name{osem_measurements_archive}
|
||||
\alias{osem_measurements_archive}
|
||||
\alias{osem_measurements_archive.sensebox}
|
||||
\title{Get day-wise measurements for a single box from the openSenseMap archive.}
|
||||
\usage{
|
||||
osem_measurements_archive(x, ...)
|
||||
|
||||
\method{osem_measurements_archive}{sensebox}(x, fromDate,
|
||||
toDate = fromDate, sensorFilter = ~T, progress = T)
|
||||
}
|
||||
\description{
|
||||
This function is significantly faster than `osem_measurements()` for large
|
||||
time-frames, as dayly CSV dumps for each sensor from
|
||||
<archive.opensensemap.org> are used.
|
||||
Note that the latest data available is from the previous day.
|
||||
By default, data for all sensors of a box is fetched, but you can select a
|
||||
subset with a `dplyr`-style NSE filter expression.
|
||||
}
|
||||
\section{Methods (by class)}{
|
||||
\itemize{
|
||||
\item \code{sensebox}: Get daywise measurements for one or
|
||||
more sensors of a single box
|
||||
}}
|
||||
|
||||
\examples{
|
||||
|
||||
\donttest{
|
||||
# fetch measurements for a single day
|
||||
box = osem_box('593bcd656ccf3b0011791f5a')
|
||||
m = osem_measurements_archive(box, as.POSIXlt('2018-09-13'))
|
||||
|
||||
# fetch measurements for a date range and selected sensors
|
||||
sensors = ~ phenomenon \%in\% c('Temperatur', 'Beleuchtungsstärke')
|
||||
m = osem_measurements_archive(box, as.POSIXlt('2018-09-01'), as.POSIXlt('2018-09-30'), sensorFilter = sensors)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
source('testhelpers.R')
|
||||
|
||||
context('osem_box_to_archivename()')
|
||||
|
||||
try({
|
||||
boxes = osem_boxes(grouptag = 'ifgi')
|
||||
box = filter(boxes, row_number() == 1)
|
||||
})
|
||||
|
||||
test_that('osem_box_to_archive_name does the correct character replacements', {
|
||||
b = data.frame(
|
||||
name = 'aA1._- äß!"?$%&/',
|
||||
X_id = 'UUID'
|
||||
)
|
||||
|
||||
archivename = opensensmapr:::osem_box_to_archivename(b)
|
||||
expect_equal(archivename, 'UUID-aA1._-__________')
|
||||
})
|
||||
|
||||
test_that('osem_box_to_archive_name works for one box', {
|
||||
if (is.null(box)) skip('no box data could be fetched')
|
||||
|
||||
archivename = opensensmapr:::osem_box_to_archivename(box)
|
||||
expect_length(archivename, 1)
|
||||
expect_type(archivename, 'character')
|
||||
})
|
||||
|
||||
test_that('osem_box_to_archive_name works for multiple boxes', {
|
||||
if (is.null(boxes)) skip('no box data available')
|
||||
|
||||
archivename = opensensmapr:::osem_box_to_archivename(boxes)
|
||||
expect_length(archivename, nrow(boxes))
|
||||
expect_type(archivename, 'character')
|
||||
})
|
||||
|
||||
context('osem_measurements_archive()')
|
||||
|
||||
test_that('osem_measurements_archive works for one box', {
|
||||
if (is.null(box)) skip('no box data could be fetched')
|
||||
|
||||
m = osem_measurements_archive(box, as.POSIXlt('2018-08-08'))
|
||||
expect_length(m, nrow(box$sensors[[1]]) + 1) # one column for each sensor + createdAt
|
||||
expect_s3_class(m, c('osem_measurements', 'tbl_df'))
|
||||
})
|
||||
|
||||
test_that('osem_measurements_archive fails for multiple boxes', {
|
||||
if (is.null(boxes)) skip('no box data available')
|
||||
|
||||
expect_error(
|
||||
osem_measurements_archive(boxes, as.POSIXlt('2018-08-08')),
|
||||
'this function only works for exactly one senseBox!'
|
||||
)
|
||||
})
|
||||
Loading…
Reference in New Issue