sources <- "NSIDC SMMR-SSM/I Nasateam near-real-time sea ice concentration"
library(blueant)
## define a local file root, this code may be used
## to identify a predictable location for this package *for a given user*
local_file_root <- rappdirs::user_data_dir(appname = "seaice")

## create the local directly if it doesn't exist
if (!file.exists(local_file_root) || file.info(local_file_root)$isdir) {
  dir.create(local_file_root ,recursive = TRUE)
}
## /home/rstudio-user/.local/share/seaice
config <- bb_config(local_file_root = local_file_root)
config <- config %>% bb_add(blueant_sources(sources))
bb_sync(config)

library(tidyverse)
library(fs)
library(bowerbird)

id_data_gouv <- "5e1f20058b4c414d3f94460d"


bb_data_gouv_source <- function(id) {
  ne_or <- function(z, or) tryCatch(if (!is.null(z) && nzchar(z)) z else or, error = function(e) or)
  jx <- jsonlite::fromJSON(paste0("https://www.data.gouv.fr/api/1/datasets/", id_data_gouv))
  #collection sizes
  csize <- tryCatch(as.numeric(format(sum(jx$resources$filesize, na.rm = TRUE)/1024^3, digits = 1)), error = function(e) NULL)
  id <- id
  description <- jx$description
  folder <- jx$slug
  doc_url <-jx$page
  postproc <- list()
  files <- paste0(fs::path_ext_remove(jx$resources$title), '.', jx$resources$format)
  bb_source(name = ne_or(jx$title, ne_or(jx$acronym, "Dataset title")),
            id = id,
            description = ne_or(jx$description, "Dataset description"),
            ##keywords = ne_or(jx$metadata$keywords, NA_character_),
            doc_url = jx$page,
            citation = paste0("See ", jx$page, " for the correct citation"), ## seems odd that this isn't part of the record
            license = ne_or(jx$license, paste0("See ", doc_url, " for license information")),
            source_url = jx$resources$latest, ## list all urls. Does this cover datasets with multiple buckets? (Are there such things?)
            method = list("bb_data_gouv_handler_get", files = files), 
            comment = "Source definition created by bb_zenodo_source",
            postprocess = postproc,
            collection_size = csize)
  
  
}

bb_data_gouv_handler_get <- function(config, verbose = TRUE, files, ...) {
  cfrow <- bowerbird:::bb_settings_to_cols(config)
  this_flags <- list(...)
  this_flags <- c(list(url = cfrow$source_url), list(force_local_filename = files),this_flags, list(verbose = verbose))
  do.call(bb_rget, this_flags)
}


src <- bb_data_gouv_source(id_data_gouv)
cf <- bb_config(local_file_root = "~")
cf <- bb_add(cf, src )
status <- bb_sync(cf, create_root = TRUE, verbose = TRUE)
#> 
#> Tue Oct 12 16:14:28 2021
#> Synchronizing dataset: Base nationale sur les intercommunalités
#> Source URL https://www.data.gouv.fr/fr/datasets/r/8af06636-9d44-4cfd-92b0-32a7e2059c75
#> --------------------------------------------------------------------------------------------
#> 
#> There was a problem synchronizing the dataset: Base nationale sur les intercommunalités.
#> The error message was: argument inutilisé (local_dir_only = TRUE) 
#> 
#> Tue Oct 12 16:14:28 2021
#> Synchronizing dataset: Base nationale sur les intercommunalités
#> Source URL https://www.data.gouv.fr/fr/datasets/r/08a78f58-81cc-43f3-b16a-f4d4a6392ab3
#> --------------------------------------------------------------------------------------------
#> 
#> There was a problem synchronizing the dataset: Base nationale sur les intercommunalités.
#> The error message was: argument inutilisé (local_dir_only = TRUE) 
#> 
#> Tue Oct 12 16:14:28 2021
#> Synchronizing dataset: Base nationale sur les intercommunalités
#> Source URL https://www.data.gouv.fr/fr/datasets/r/ed17ac9b-2025-482b-bda4-f9d667644b8b
#> --------------------------------------------------------------------------------------------
#> 
#> There was a problem synchronizing the dataset: Base nationale sur les intercommunalités.
#> The error message was: argument inutilisé (local_dir_only = TRUE) 
#> 
#> Tue Oct 12 16:14:28 2021
#> Synchronizing dataset: Base nationale sur les intercommunalités
#> Source URL https://www.data.gouv.fr/fr/datasets/r/e655f973-fb12-4771-9723-b1e4a2b086b2
#> --------------------------------------------------------------------------------------------
#> 
#> There was a problem synchronizing the dataset: Base nationale sur les intercommunalités.
#> The error message was: argument inutilisé (local_dir_only = TRUE) 
#> 
#> Tue Oct 12 16:14:28 2021
#> Synchronizing dataset: Base nationale sur les intercommunalités
#> Source URL https://www.data.gouv.fr/fr/datasets/r/5c878e99-96a9-48dc-b349-03878ff34392
#> --------------------------------------------------------------------------------------------
#> 
#> There was a problem synchronizing the dataset: Base nationale sur les intercommunalités.
#> The error message was: argument inutilisé (local_dir_only = TRUE) 
#> 
#> Tue Oct 12 16:14:28 2021
#> Synchronizing dataset: Base nationale sur les intercommunalités
#> Source URL https://www.data.gouv.fr/fr/datasets/r/85d11f2d-f7cd-469b-89e5-b210d2658e4f
#> --------------------------------------------------------------------------------------------
#> 
#> There was a problem synchronizing the dataset: Base nationale sur les intercommunalités.
#> The error message was: argument inutilisé (local_dir_only = TRUE) 
#> 
#> Tue Oct 12 16:14:28 2021
#> Synchronizing dataset: Base nationale sur les intercommunalités
#> Source URL https://www.data.gouv.fr/fr/datasets/r/d743f361-0376-4d84-bc5b-7e41a5cb86c6
#> --------------------------------------------------------------------------------------------
#> 
#> There was a problem synchronizing the dataset: Base nationale sur les intercommunalités.
#> The error message was: argument inutilisé (local_dir_only = TRUE)

td <- tempdir()  ## strictly, user should specify this explicitly for control over file system
cf <- bb_config(local_file_root = td)

## Bowerbird must then be told which data sources to synchronize. 
## Let’s use data from the Australian 2016 federal election, which is provided as one 
## of the example data sources:
my_source <- subset(bb_example_sources(),id=="aus-election-house-2016")

## add this data source to the configuration
cf <- bb_add(cf,my_source)
## Once the configuration has been defined and the data source added to it, run the sync process:
status <- bb_sync(cf)

## see the fruits of our labour (files are CSVs in this data source)
head(list.files(td, recursive = TRUE, pattern = "csv$"))

## we can run this at any later time and our repository will update if the source has changed
status2 <- bb_sync(cf)

mysrc <- mysrc %>%
  mutate(method=c(method,list(accept_regex="/2017/")))

library(bowerbird)
my_directory <- "~/my/data/directory"
cf <- bb_config(local_file_root = my_directory)
my_source <- blueant::sources("CMEMS global gridded SSH near-real-time")
library(dplyr)
my_source <- my_source %>%
  mutate(method= list(c("bb_handler_wget", level = 3, 
                      accept_regex = "nrt_global_allsat_phy_l4_2018.*.nc")))

library(bowerbird)
modis <- bb_source(
  name = "MODIS MOD14 C6 L3 Fire Product",
  id = "c6-mcd14ml",
  description = "MODIS Monthly Fire Product",
  doc_url = "https://lpdaac.usgs.gov/documents/88/MOD14_User_Guide_v6.pdf/",
  citation = "https://doi.org/10.1029/2005JG000142",
  source_url = c("ftp://fuoco.geog.umd.edu/modis/C6/mcd14ml/"),
  license = "CC-BY",
  user = "fire",
  password = "burnt",
  method = list("bb_handler_rget", level = 1, accept_download = "\\.gz$"),
  collection_size=NA)

my_directory <- rappdirs::user_data_dir("bowerbird")
dir.create(my_directory, F, T)
cf <- bb_config(local_file_root = my_directory)
cf <- bb_add(cf, modis)
status <- bb_sync(cf, verbose = TRUE)

b <- "http://biogeo.ucdavis.edu/data/gadm2.8/rds/%s_adm0.rds"
gadm0 <- sprintf(b, raster::getData("ISO3")$ISO3)

gadm.rds <- bb_source(
  name="GADM maps and data in RDS format",
  id="gadm-maps-rdb",
  description="GADM provides maps and spatial data for all countries and their sub-divisions.",
  doc_url="http://www.gadm.org",
  citation="http://gadm.org/about.html",
  source_url= gadm0,
  license="http://gadm.org/license.html",
  method=list("bb_handler_wget",level=1, robots_off=TRUE),
 collection_size= 0.1,
  access_function = "base::readRDS",
  data_group="Administrative")

gadm <- bb_source(
  name="GADM maps and data in ESRI Geodatabase",
  id="gadm-maps-gdb",
  description="GADM provides maps and spatial data for all countries and their sub-divisions.",
  doc_url="http://www.gadm.org",
  citation="http://gadm.org/about.html",
  source_url=c("http://biogeo.ucdavis.edu/data/gadm2.8/gadm28.gdb.zip"),
  license="http://gadm.org/license.html",
  method=list("bb_handler_wget",recursive=TRUE,level=1, robots_off=TRUE),
  postprocess=list("bb_unzip"),
  collection_size= 1,
  access_function = "sf::read_sf",
  data_group="Administrative")