[ceph.git] / ceph / src / arrow / r / tests / testthat / test-filesystem.R

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.


test_that("LocalFilesystem", {
  fs <- LocalFileSystem$create()
  expect_identical(fs$type_name, "local")
  DESCRIPTION <- system.file("DESCRIPTION", package = "arrow")
  info <- fs$GetFileInfo(DESCRIPTION)[[1]]
  expect_equal(info$base_name(), "DESCRIPTION")
  expect_equal(info$extension(), "")
  expect_equal(info$type, FileType$File)
  expect_equal(info$path, DESCRIPTION)
  info <- file.info(DESCRIPTION)

  expect_equal(info$size, info$size)
  # This fails due to a subsecond difference on Appveyor on Windows with R 3.3 only
  # So add a greater tolerance to allow for that
  expect_equal(info$mtime, info$mtime, tolerance = 1)

  tf <- tempfile(fileext = ".txt")
  fs$CopyFile(DESCRIPTION, tf)
  info <- fs$GetFileInfo(tf)[[1]]
  expect_equal(info$extension(), "txt")
  expect_equal(info$size, info$size)
  expect_equal(readLines(DESCRIPTION), readLines(tf))

  tf2 <- tempfile(fileext = ".txt")
  fs$Move(tf, tf2)
  infos <- fs$GetFileInfo(c(tf, tf2, dirname(tf)))
  expect_equal(infos[[1]]$type, FileType$NotFound)
  expect_equal(infos[[2]]$type, FileType$File)
  expect_equal(infos[[3]]$type, FileType$Directory)

  fs$DeleteFile(tf2)
  expect_equal(fs$GetFileInfo(tf2)[[1L]]$type, FileType$NotFound)
  expect_true(!file.exists(tf2))

  expect_equal(fs$GetFileInfo(tf)[[1L]]$type, FileType$NotFound)
  expect_true(!file.exists(tf))

  td <- tempfile()
  fs$CreateDir(td)
  expect_equal(fs$GetFileInfo(td)[[1L]]$type, FileType$Directory)
  fs$CopyFile(DESCRIPTION, file.path(td, "DESCRIPTION"))
  fs$DeleteDirContents(td)
  expect_equal(length(dir(td)), 0L)
  fs$DeleteDir(td)
  expect_equal(fs$GetFileInfo(td)[[1L]]$type, FileType$NotFound)

  tf3 <- tempfile()
  os <- fs$OpenOutputStream(path = tf3)
  bytes <- as.raw(1:40)
  os$write(bytes)
  os$close()

  is <- fs$OpenInputStream(tf3)
  buf <- is$Read(40)
  expect_equal(buf$data(), bytes)
  is$close()
})

test_that("SubTreeFilesystem", {
  dir.create(td <- tempfile())
  DESCRIPTION <- system.file("DESCRIPTION", package = "arrow")
  file.copy(DESCRIPTION, file.path(td, "DESCRIPTION"))

  st_fs <- SubTreeFileSystem$create(td)
  expect_r6_class(st_fs, "SubTreeFileSystem")
  expect_r6_class(st_fs, "FileSystem")
  expect_r6_class(st_fs$base_fs, "LocalFileSystem")
  expect_identical(
    capture.output(print(st_fs)),
    paste0("SubTreeFileSystem: ", "file://", st_fs$base_path)
  )

  # FIXME windows has a trailing slash for one but not the other
  # expect_identical(normalizePath(st_fs$base_path), normalizePath(td)) # nolint

  st_fs$CreateDir("test")
  st_fs$CopyFile("DESCRIPTION", "DESC.txt")
  infos <- st_fs$GetFileInfo(c("DESCRIPTION", "test", "nope", "DESC.txt"))
  expect_equal(infos[[1L]]$type, FileType$File)
  expect_equal(infos[[2L]]$type, FileType$Directory)
  expect_equal(infos[[3L]]$type, FileType$NotFound)
  expect_equal(infos[[4L]]$type, FileType$File)
  expect_equal(infos[[4L]]$extension(), "txt")

  local_fs <- LocalFileSystem$create()
  local_fs$DeleteDirContents(td)
  infos <- st_fs$GetFileInfo(c("DESCRIPTION", "test", "nope", "DESC.txt"))
  expect_equal(infos[[1L]]$type, FileType$NotFound)
  expect_equal(infos[[2L]]$type, FileType$NotFound)
  expect_equal(infos[[3L]]$type, FileType$NotFound)
  expect_equal(infos[[4L]]$type, FileType$NotFound)
})

test_that("LocalFileSystem + Selector", {
  fs <- LocalFileSystem$create()
  dir.create(td <- tempfile())
  writeLines("blah blah", file.path(td, "one.txt"))
  writeLines("yada yada", file.path(td, "two.txt"))
  dir.create(file.path(td, "dir"))
  writeLines("...", file.path(td, "dir", "three.txt"))

  selector <- FileSelector$create(td, recursive = TRUE)
  infos <- fs$GetFileInfo(selector)
  expect_equal(length(infos), 4L)
  types <- sapply(infos, function(.x) .x$type)
  expect_equal(sum(types == FileType$File), 3L)
  expect_equal(sum(types == FileType$Directory), 1L)

  selector <- FileSelector$create(td, recursive = FALSE)
  infos <- fs$GetFileInfo(selector)
  expect_equal(length(infos), 3L)
  types <- sapply(infos, function(.x) .x$type)
  expect_equal(sum(types == FileType$File), 2L)
  expect_equal(sum(types == FileType$Directory), 1L)
})

test_that("FileSystem$from_uri", {
  skip_on_cran()
  skip_if_not_available("s3")
  skip_if_offline()
  fs_and_path <- FileSystem$from_uri("s3://ursa-labs-taxi-data")
  expect_r6_class(fs_and_path$fs, "S3FileSystem")
  expect_identical(fs_and_path$fs$region, "us-east-2")
})

test_that("SubTreeFileSystem$create() with URI", {
  skip_on_cran()
  skip_if_not_available("s3")
  skip_if_offline()
  fs <- SubTreeFileSystem$create("s3://ursa-labs-taxi-data")
  expect_r6_class(fs, "SubTreeFileSystem")
  expect_identical(
    capture.output(print(fs)),
    "SubTreeFileSystem: s3://ursa-labs-taxi-data/"
  )
})

test_that("S3FileSystem", {
  skip_on_cran()
  skip_if_not_available("s3")
  skip_if_offline()
  s3fs <- S3FileSystem$create()
  expect_r6_class(s3fs, "S3FileSystem")
})

test_that("s3_bucket", {
  skip_on_cran()
  skip_if_not_available("s3")
  skip_if_offline()
  bucket <- s3_bucket("ursa-labs-r-test")
  expect_r6_class(bucket, "SubTreeFileSystem")
  expect_r6_class(bucket$base_fs, "S3FileSystem")
  expect_identical(bucket$region, "us-west-2")
  expect_identical(
    capture.output(print(bucket)),
    "SubTreeFileSystem: s3://ursa-labs-r-test/"
  )
  skip_on_os("windows") # FIXME
  expect_identical(bucket$base_path, "ursa-labs-r-test/")
})
Commit	Line	Data
1d09f67e TL	1	# Licensed to the Apache Software Foundation (ASF) under one
	2	# or more contributor license agreements. See the NOTICE file
	3	# distributed with this work for additional information
	4	# regarding copyright ownership. The ASF licenses this file
	5	# to you under the Apache License, Version 2.0 (the
	6	# "License"); you may not use this file except in compliance
	7	# with the License. You may obtain a copy of the License at
	8	#
	9	# http://www.apache.org/licenses/LICENSE-2.0
	10	#
	11	# Unless required by applicable law or agreed to in writing,
	12	# software distributed under the License is distributed on an
	13	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	14	# KIND, either express or implied. See the License for the
	15	# specific language governing permissions and limitations
	16	# under the License.
	17
	18
	19	test_that("LocalFilesystem", {
	20	fs <- LocalFileSystem$create()
	21	expect_identical(fs$type_name, "local")
	22	DESCRIPTION <- system.file("DESCRIPTION", package = "arrow")
	23	info <- fs$GetFileInfo(DESCRIPTION)[[1]]
	24	expect_equal(info$base_name(), "DESCRIPTION")
	25	expect_equal(info$extension(), "")
	26	expect_equal(info$type, FileType$File)
	27	expect_equal(info$path, DESCRIPTION)
	28	info <- file.info(DESCRIPTION)
	29
	30	expect_equal(info$size, info$size)
	31	# This fails due to a subsecond difference on Appveyor on Windows with R 3.3 only
	32	# So add a greater tolerance to allow for that
	33	expect_equal(info$mtime, info$mtime, tolerance = 1)
	34
	35	tf <- tempfile(fileext = ".txt")
	36	fs$CopyFile(DESCRIPTION, tf)
	37	info <- fs$GetFileInfo(tf)[[1]]
	38	expect_equal(info$extension(), "txt")
	39	expect_equal(info$size, info$size)
	40	expect_equal(readLines(DESCRIPTION), readLines(tf))
	41
	42	tf2 <- tempfile(fileext = ".txt")
	43	fs$Move(tf, tf2)
	44	infos <- fs$GetFileInfo(c(tf, tf2, dirname(tf)))
	45	expect_equal(infos[[1]]$type, FileType$NotFound)
	46	expect_equal(infos[[2]]$type, FileType$File)
	47	expect_equal(infos[[3]]$type, FileType$Directory)
	48
	49	fs$DeleteFile(tf2)
	50	expect_equal(fs$GetFileInfo(tf2)[[1L]]$type, FileType$NotFound)
	51	expect_true(!file.exists(tf2))
	52
	53	expect_equal(fs$GetFileInfo(tf)[[1L]]$type, FileType$NotFound)
	54	expect_true(!file.exists(tf))
	55
	56	td <- tempfile()
	57	fs$CreateDir(td)
	58	expect_equal(fs$GetFileInfo(td)[[1L]]$type, FileType$Directory)
	59	fs$CopyFile(DESCRIPTION, file.path(td, "DESCRIPTION"))
	60	fs$DeleteDirContents(td)
	61	expect_equal(length(dir(td)), 0L)
	62	fs$DeleteDir(td)
	63	expect_equal(fs$GetFileInfo(td)[[1L]]$type, FileType$NotFound)
	64
65	tf3 <- tempfile()
66	os <- fs$OpenOutputStream(path = tf3)
67	bytes <- as.raw(1:40)
68	os$write(bytes)
69	os$close()
70
71	is <- fs$OpenInputStream(tf3)
72	buf <- is$Read(40)
73	expect_equal(buf$data(), bytes)
74	is$close()
75	})
76
77	test_that("SubTreeFilesystem", {
78	dir.create(td <- tempfile())
79	DESCRIPTION <- system.file("DESCRIPTION", package = "arrow")
80	file.copy(DESCRIPTION, file.path(td, "DESCRIPTION"))
81
82	st_fs <- SubTreeFileSystem$create(td)
83	expect_r6_class(st_fs, "SubTreeFileSystem")
84	expect_r6_class(st_fs, "FileSystem")
85	expect_r6_class(st_fs$base_fs, "LocalFileSystem")
86	expect_identical(
87	capture.output(print(st_fs)),
88	paste0("SubTreeFileSystem: ", "file://", st_fs$base_path)
89	)
90
91	# FIXME windows has a trailing slash for one but not the other
92	# expect_identical(normalizePath(st_fs$base_path), normalizePath(td)) # nolint
93
94	st_fs$CreateDir("test")
95	st_fs$CopyFile("DESCRIPTION", "DESC.txt")
96	infos <- st_fs$GetFileInfo(c("DESCRIPTION", "test", "nope", "DESC.txt"))
97	expect_equal(infos[[1L]]$type, FileType$File)
98	expect_equal(infos[[2L]]$type, FileType$Directory)
99	expect_equal(infos[[3L]]$type, FileType$NotFound)
100	expect_equal(infos[[4L]]$type, FileType$File)
101	expect_equal(infos[[4L]]$extension(), "txt")
102
103	local_fs <- LocalFileSystem$create()
104	local_fs$DeleteDirContents(td)
105	infos <- st_fs$GetFileInfo(c("DESCRIPTION", "test", "nope", "DESC.txt"))
106	expect_equal(infos[[1L]]$type, FileType$NotFound)
107	expect_equal(infos[[2L]]$type, FileType$NotFound)
108	expect_equal(infos[[3L]]$type, FileType$NotFound)
109	expect_equal(infos[[4L]]$type, FileType$NotFound)
110	})
111
112	test_that("LocalFileSystem + Selector", {
113	fs <- LocalFileSystem$create()
114	dir.create(td <- tempfile())
115	writeLines("blah blah", file.path(td, "one.txt"))
116	writeLines("yada yada", file.path(td, "two.txt"))
117	dir.create(file.path(td, "dir"))
118	writeLines("...", file.path(td, "dir", "three.txt"))
119
120	selector <- FileSelector$create(td, recursive = TRUE)
121	infos <- fs$GetFileInfo(selector)
122	expect_equal(length(infos), 4L)
123	types <- sapply(infos, function(.x) .x$type)
124	expect_equal(sum(types == FileType$File), 3L)
125	expect_equal(sum(types == FileType$Directory), 1L)
126
127	selector <- FileSelector$create(td, recursive = FALSE)
128	infos <- fs$GetFileInfo(selector)
129	expect_equal(length(infos), 3L)
130	types <- sapply(infos, function(.x) .x$type)
131	expect_equal(sum(types == FileType$File), 2L)
132	expect_equal(sum(types == FileType$Directory), 1L)
133	})
134
135	test_that("FileSystem$from_uri", {
136	skip_on_cran()
137	skip_if_not_available("s3")
138	skip_if_offline()
139	fs_and_path <- FileSystem$from_uri("s3://ursa-labs-taxi-data")
140	expect_r6_class(fs_and_path$fs, "S3FileSystem")
141	expect_identical(fs_and_path$fs$region, "us-east-2")
142	})
143
144	test_that("SubTreeFileSystem$create() with URI", {
145	skip_on_cran()
146	skip_if_not_available("s3")
147	skip_if_offline()
148	fs <- SubTreeFileSystem$create("s3://ursa-labs-taxi-data")
149	expect_r6_class(fs, "SubTreeFileSystem")
150	expect_identical(
151	capture.output(print(fs)),
152	"SubTreeFileSystem: s3://ursa-labs-taxi-data/"
153	)
154	})
155
156	test_that("S3FileSystem", {
157	skip_on_cran()
158	skip_if_not_available("s3")
159	skip_if_offline()
160	s3fs <- S3FileSystem$create()
161	expect_r6_class(s3fs, "S3FileSystem")
162	})
163
164	test_that("s3_bucket", {
165	skip_on_cran()
166	skip_if_not_available("s3")
167	skip_if_offline()
168	bucket <- s3_bucket("ursa-labs-r-test")
169	expect_r6_class(bucket, "SubTreeFileSystem")
170	expect_r6_class(bucket$base_fs, "S3FileSystem")
171	expect_identical(bucket$region, "us-west-2")
172	expect_identical(
173	capture.output(print(bucket)),
174	"SubTreeFileSystem: s3://ursa-labs-r-test/"
175	)
176	skip_on_os("windows") # FIXME
177	expect_identical(bucket$base_path, "ursa-labs-r-test/")
178	})