wehub/tensorflow--tensorflow
1
0
Fork 0
Code Issues Pull Requests Actions 3 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
cffconvert / validate (push) Has been skipped
Details
License Check / license-check (push) Failing after 2s
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:14:16 +08:00
commit 8a852e4b4e
36502 changed files with 9277225 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tensorflow/core/lib/bfloat16/BUILD
+33
View File
@@ -0,0 +1,33 @@
# load(
# "//tensorflow/core/platform:rules_cc.bzl",
# "cc_library",
# )
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = [
"//tensorflow:__subpackages__",
],
licenses = ["notice"],
)
# cc_library(
# name = "bfloat16",
# hdrs = ["bfloat16.h"],
# deps = [
# "@eigen_archive//:eigen3",
# "//tensorflow/core/platform:byte_order",
# ],
# )
# # Export source files needed for mobile builds, which do not use granular targets.
# filegroup(
# name = "mobile_srcs_no_runtime",
# srcs = [
# "bfloat16.h",
# ],
# )
exports_files(
["bfloat16.h"],
)
tensorflow/core/lib/bfloat16/bfloat16.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_BFLOAT16_BFLOAT16_H_
#define TENSORFLOW_CORE_LIB_BFLOAT16_BFLOAT16_H_
#include "tensorflow/core/platform/bfloat16.h"
#endif // TENSORFLOW_CORE_LIB_BFLOAT16_BFLOAT16_H_
tensorflow/core/lib/bmp/BUILD
+13
View File
@@ -0,0 +1,13 @@
# Description:
# bmp test data package alias.
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
alias(
name = "bmp_testdata",
actual = "//tensorflow/core/lib/bmp/testdata:bmp_testdata",
visibility = ["//visibility:public"],
)
tensorflow/core/lib/bmp/testdata/BUILD
Vendored
+25
View File
@@ -0,0 +1,25 @@
# Description:
# bmp test data packages.
load("//tensorflow:tensorflow.default.bzl", "filegroup")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
filegroup(
name = "bmp_testdata",
srcs = [
# BMP data
"lena.bmp",
"rgb_small.bmp",
"rgb_small_255.bmp",
"rgba_small.bmp",
"rgba_small_255.bmp",
"grayscale_small.bmp",
"grayscale_small_3channels.bmp",
"grayscale_small_4channels.bmp",
],
visibility = ["//visibility:public"],
)
tensorflow/core/lib/bmp/testdata/grayscale_small.bmp
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.2 KiB

tensorflow/core/lib/bmp/testdata/grayscale_small_3channels.bmp
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 630 B

tensorflow/core/lib/bmp/testdata/grayscale_small_4channels.bmp
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 822 B

tensorflow/core/lib/bmp/testdata/lena.bmp
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.1 KiB

tensorflow/core/lib/bmp/testdata/rgb_small.bmp
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 630 B

tensorflow/core/lib/bmp/testdata/rgb_small_255.bmp
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 714 B

tensorflow/core/lib/bmp/testdata/rgba_small.bmp
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 906 B

tensorflow/core/lib/bmp/testdata/rgba_small_255.bmp
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 906 B

tensorflow/core/lib/core/BUILD
+312
View File
@@ -0,0 +1,312 @@
load("//tensorflow:tensorflow.bzl", "tf_cc_test")
load("//tensorflow:tensorflow.default.bzl", "filegroup")
load("//tensorflow/core/platform:build_config.bzl", "tf_proto_library")
load(
"//tensorflow/core/platform:rules_cc.bzl",
"cc_library",
)
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = [
"//tensorflow/core:__subpackages__",
"//tensorflow/python/profiler/internal:__pkg__",
],
licenses = ["notice"],
)
# Todo(bmzhao): Remaining targets to add to this BUILD file are all tests.
cc_library(
name = "arena",
srcs = ["arena.cc"],
hdrs = ["arena.h"],
deps = [
"//tensorflow/core/lib/math:math_util",
"//tensorflow/core/platform:logging",
"//tensorflow/core/platform:macros",
"//tensorflow/core/platform:platform_port",
"//tensorflow/core/platform:types",
],
)
tf_cc_test(
name = "arena_test",
srcs = ["arena_test.cc"],
deps = [
":arena",
"//tensorflow/core:test",
"//tensorflow/core/platform:macros",
"@xla//xla/tsl/platform:test_main",
],
)
cc_library(
name = "bitmap",
hdrs = ["bitmap.h"],
deps = [
"@xla//xla/tsl/lib/core:bitmap",
],
alwayslink = 1,
)
cc_library(
name = "bits",
hdrs = ["bits.h"],
deps = [
"//tensorflow/core/platform:logging",
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/core:bits",
],
)
cc_library(
name = "coding",
hdrs = ["coding.h"],
deps = [
"//tensorflow/core/platform:coding",
],
)
tf_cc_test(
name = "coding_test",
srcs = ["coding_test.cc"],
deps = [
":coding",
"//tensorflow/core:test",
"@xla//xla/tsl/platform:test_main",
],
)
cc_library(
name = "errors",
hdrs = ["errors.h"],
deps = [
"//tensorflow/core/platform:errors",
],
)
cc_library(
name = "notification",
hdrs = ["notification.h"],
)
cc_library(
name = "raw_coding",
hdrs = ["raw_coding.h"],
deps = [
"//tensorflow/core/platform:raw_coding",
],
)
cc_library(
name = "refcount",
hdrs = ["refcount.h"],
deps = ["//tensorflow/core/platform:refcount"],
)
cc_library(
name = "status",
hdrs = ["status.h"],
deps = [
"//tensorflow/core/platform:status",
],
)
tf_cc_test(
name = "status_test",
srcs = ["status_test.cc"],
deps = [
":errors",
":status",
"//tensorflow/core:test",
"@com_google_absl//absl/strings",
"@xla//xla/tsl/platform:test_main",
],
)
cc_library(
name = "stringpiece",
hdrs = ["stringpiece.h"],
deps = [
"//tensorflow/core/platform:stringpiece",
],
)
cc_library(
name = "threadpool",
hdrs = ["threadpool.h"],
deps = [
"//tensorflow/core/platform:env",
],
)
tf_cc_test(
name = "threadpool_test",
srcs = ["threadpool_test.cc"],
deps = [
":threadpool",
"//tensorflow/core:framework_lite",
"//tensorflow/core:lib",
"//tensorflow/core:test",
"@com_google_absl//absl/synchronization",
"@com_google_absl//absl/time",
"@xla//xla/tsl/platform:test_main",
],
)
cc_library(
name = "threadpool_interface",
hdrs = ["threadpool_interface.h"],
deps = [
"//tensorflow/core/platform:threadpool_interface",
],
)
cc_library(
name = "threadpool_options",
hdrs = ["threadpool_options.h"],
deps = [
"//tensorflow/core/platform:threadpool_options",
],
)
tf_proto_library(
name = "error_codes_proto",
srcs = ["error_codes.proto"],
make_default_target_header_only = True,
protodeps = [
"@xla//xla/tsl/protobuf:error_codes_proto_impl",
],
exports = ["@xla//xla/tsl/protobuf:error_codes_proto_impl"],
)
# Export source files needed for mobile builds, which do not use granular targets.
filegroup(
name = "mobile_hdrs_no_runtime",
srcs = [
"coding.h",
"errors.h",
"refcount.h",
"status.h",
"stringpiece.h",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "mobile_srcs_only_runtime",
srcs = [
"arena.cc",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "mobile_hdrs_only_runtime",
srcs = [
"arena.h",
"bitmap.h",
"bits.h",
"notification.h",
"threadpool.h",
"threadpool_interface.h",
"threadpool_options.h",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_core_all_headers",
srcs = [
"arena.h",
"bitmap.h",
"bits.h",
"coding.h",
"errors.h",
"notification.h",
"raw_coding.h",
"refcount.h",
"status.h",
"status_test_util.h",
"stringpiece.h",
"threadpool.h",
"threadpool_interface.h",
"threadpool_options.h",
"@xla//xla/tsl/lib/core:legacy_lib_core_all_headers",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_internal_core_headers",
srcs = [
"refcount.h",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_core_status_header",
srcs = [
"status.h",
],
visibility = ["//visibility:private"],
)
filegroup(
name = "legacy_lib_core_status_test_util_header",
srcs = [
"status_test_util.h",
"@xla//xla/tsl/lib/core:legacy_lib_core_status_test_util_header",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_core_stringpiece_header",
srcs = [
"stringpiece.h",
],
visibility = [
"//tensorflow/core:__pkg__",
"//tensorflow/core/lib/gif:__pkg__",
"//tensorflow/core/lib/jpeg:__pkg__",
],
)
filegroup(
name = "legacy_lib_proto_parsing_headers",
srcs = [
"errors.h",
"status.h",
"stringpiece.h",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_core_headers",
srcs = [
"arena.h",
"bitmap.h",
"bits.h",
"coding.h",
"errors.h",
"notification.h",
"raw_coding.h",
"refcount.h",
"status.h",
"stringpiece.h",
"threadpool.h",
"threadpool_interface.h",
"threadpool_options.h",
],
visibility = ["//tensorflow/core:__pkg__"],
)
# This is needed because of how tf_android_core_proto_sources parses proto paths.
exports_files(
srcs = ["error_codes.proto"],
visibility = ["//tensorflow/core:__pkg__"],
)
tensorflow/core/lib/core/arena.cc
+252
View File
@@ -0,0 +1,252 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This approach to arenas overcomes many of the limitations described
// in the "Specialized allocators" section of
// http://www.pdos.lcs.mit.edu/~dm/c++-new.html
//
// A somewhat similar approach to Gladiator, but for heap-detection, was
// suggested by Ron van der Wal and Scott Meyers at
// http://www.aristeia.com/BookErrata/M27Comments_frames.html
#include "tensorflow/core/lib/core/arena.h"
#include <assert.h>
#include <algorithm>
#include <vector>
#include "tensorflow/core/lib/math/math_util.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mem.h"
namespace tensorflow {
namespace core {
// ----------------------------------------------------------------------
// Arena::Arena()
// Arena::~Arena()
// Destroying the arena automatically calls Reset()
// ----------------------------------------------------------------------
Arena::Arena(const size_t block_size)
: remaining_(0),
block_size_(block_size),
freestart_(nullptr), // set for real in Reset()
blocks_alloced_(1),
overflow_blocks_(nullptr) {
assert(block_size > kDefaultAlignment);
first_blocks_[0].mem = reinterpret_cast<char*>(tsl::port::AlignedMalloc(
block_size_, static_cast<std::align_val_t>(sizeof(void*))));
first_blocks_[0].size = block_size_;
Reset();
}
Arena::~Arena() {
FreeBlocks();
assert(overflow_blocks_ == nullptr); // FreeBlocks() should do that
// The first X blocks stay allocated always by default. Delete them now.
for (size_t i = 0; i < blocks_alloced_; ++i) {
port::AlignedFree(first_blocks_[i].mem);
}
}
// Returns true iff it advances freestart_ to the first position
// satisfying alignment without exhausting the current block.
bool Arena::SatisfyAlignment(size_t alignment) {
const size_t overage = reinterpret_cast<size_t>(freestart_) & (alignment - 1);
if (overage > 0) {
const size_t waste = alignment - overage;
if (waste >= remaining_) {
return false;
}
freestart_ += waste;
remaining_ -= waste;
}
DCHECK_EQ(size_t{0}, reinterpret_cast<size_t>(freestart_) & (alignment - 1));
return true;
}
// ----------------------------------------------------------------------
// Arena::Reset()
// Clears all the memory an arena is using.
// ----------------------------------------------------------------------
void Arena::Reset() {
FreeBlocks();
freestart_ = first_blocks_[0].mem;
remaining_ = first_blocks_[0].size;
// There is no guarantee the first block is properly aligned, so
// enforce that now.
CHECK(SatisfyAlignment(kDefaultAlignment));
freestart_when_empty_ = freestart_;
}
// ----------------------------------------------------------------------
// Arena::MakeNewBlock()
// Our sbrk() equivalent. We always make blocks of the same size
// (though GetMemory() can also make a new block for really big
// data.
// ----------------------------------------------------------------------
void Arena::MakeNewBlock(const uint32_t alignment) {
AllocatedBlock* block = AllocNewBlock(block_size_, alignment);
freestart_ = block->mem;
remaining_ = block->size;
CHECK(SatisfyAlignment(alignment));
}
static uint32_t LeastCommonMultiple(uint32_t a, uint32_t b) {
if (a > b) {
return (a / MathUtil::GCD<uint32_t>(a, b)) * b;
} else if (a < b) {
return (b / MathUtil::GCD<uint32_t>(b, a)) * a;
} else {
return a;
}
}
// -------------------------------------------------------------
// Arena::AllocNewBlock()
// Adds and returns an AllocatedBlock.
// The returned AllocatedBlock* is valid until the next call
// to AllocNewBlock or Reset. (i.e. anything that might
// affect overflow_blocks_).
// -------------------------------------------------------------
Arena::AllocatedBlock* Arena::AllocNewBlock(const size_t block_size,
const uint32_t alignment) {
AllocatedBlock* block;
// Find the next block.
if (blocks_alloced_ < TF_ARRAYSIZE(first_blocks_)) {
// Use one of the pre-allocated blocks
block = &first_blocks_[blocks_alloced_++];
} else { // oops, out of space, move to the vector
if (overflow_blocks_ == nullptr)
overflow_blocks_ = new std::vector<AllocatedBlock>;
// Adds another block to the vector.
overflow_blocks_->resize(overflow_blocks_->size() + 1);
// block points to the last block of the vector.
block = &overflow_blocks_->back();
}
// NOTE(tucker): this utility is made slightly more complex by
// not disallowing the case where alignment > block_size.
// Can we, without breaking existing code?
// Must be a multiple of kDefaultAlignment, unless requested
// alignment is 1, in which case we don't care at all.
uint32_t adjusted_alignment =
(alignment > 1 ? LeastCommonMultiple(alignment, kDefaultAlignment) : 1);
// Required minimum alignment for port::AlignedMalloc().
adjusted_alignment =
std::max(adjusted_alignment, static_cast<uint32_t>(sizeof(void*)));
CHECK_LE(adjusted_alignment, static_cast<uint32_t>(1 << 20))
<< "Alignment on boundaries greater than 1MB not supported.";
// If block_size > alignment we force block_size to be a multiple
// of alignment; if block_size < alignment we make no adjustment.
size_t adjusted_block_size = block_size;
if (adjusted_block_size > adjusted_alignment) {
const uint32_t excess = adjusted_block_size % adjusted_alignment;
adjusted_block_size += (excess > 0 ? adjusted_alignment - excess : 0);
}
block->mem = reinterpret_cast<char*>(tsl::port::AlignedMalloc(
adjusted_block_size, static_cast<std::align_val_t>(adjusted_alignment)));
block->size = adjusted_block_size;
CHECK(nullptr != block->mem) << "block_size=" << block_size
<< " adjusted_block_size=" << adjusted_block_size
<< " alignment=" << alignment
<< " adjusted_alignment=" << adjusted_alignment;
return block;
}
// ----------------------------------------------------------------------
// Arena::GetMemoryFallback()
// We take memory out of our pool, aligned on the byte boundary
// requested. If we don't have space in our current pool, we
// allocate a new block (wasting the remaining space in the
// current block) and give you that. If your memory needs are
// too big for a single block, we make a special your-memory-only
// allocation -- this is equivalent to not using the arena at all.
// ----------------------------------------------------------------------
void* Arena::GetMemoryFallback(const size_t size, const int alignment) {
if (0 == size) {
return nullptr; // stl/stl_alloc.h says this is okay
}
// alignment must be a positive power of 2.
CHECK(alignment > 0 && 0 == (alignment & (alignment - 1)));
// If the object is more than a quarter of the block size, allocate
// it separately to avoid wasting too much space in leftover bytes.
if (block_size_ == 0 || size > block_size_ / 4) {
return AllocNewBlock(size, alignment)->mem;
}
// Enforce alignment on freestart_ then check for adequate space,
// which may require starting a new block.
if (!SatisfyAlignment(alignment) || size > remaining_) {
MakeNewBlock(alignment);
}
CHECK_LE(size, remaining_);
remaining_ -= size;
void* result = freestart_;
freestart_ += size;
return result;
}
// ----------------------------------------------------------------------
// Arena::ReturnMemoryFallback()
// Arena::FreeBlocks()
// Unlike GetMemory(), which does actual work, ReturnMemory() is a
// no-op: we don't "free" memory until Reset() is called. We do
// update some stats, though. Note we do no checking that the
// pointer you pass in was actually allocated by us, or that it
// was allocated for the size you say, so be careful here!
// FreeBlocks() does the work for Reset(), actually freeing all
// memory allocated in one fell swoop.
// ----------------------------------------------------------------------
void Arena::FreeBlocks() {
for (size_t i = 1; i < blocks_alloced_; ++i) { // keep first block allocated
port::AlignedFree(first_blocks_[i].mem);
first_blocks_[i].mem = nullptr;
first_blocks_[i].size = 0;
}
blocks_alloced_ = 1;
if (overflow_blocks_ != nullptr) {
std::vector<AllocatedBlock>::iterator it;
for (it = overflow_blocks_->begin(); it != overflow_blocks_->end(); ++it) {
port::AlignedFree(it->mem);
}
delete overflow_blocks_; // These should be used very rarely
overflow_blocks_ = nullptr;
}
}
} // namespace core
} // namespace tensorflow
tensorflow/core/lib/core/arena.h
+111
View File
@@ -0,0 +1,111 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// TODO(vrv): Switch this to an open-sourced version of Arena.
#ifndef TENSORFLOW_CORE_LIB_CORE_ARENA_H_
#define TENSORFLOW_CORE_LIB_CORE_ARENA_H_
#include <assert.h>
#include <vector>
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace core {
// This class is "thread-compatible": different threads can access the
// arena at the same time without locking, as long as they use only
// const methods.
class Arena {
public:
// Allocates a thread-compatible arena with the specified block size.
explicit Arena(const size_t block_size);
~Arena();
char* Alloc(const size_t size) {
return reinterpret_cast<char*>(GetMemory(size, 1));
}
char* AllocAligned(const size_t size, const size_t alignment) {
return reinterpret_cast<char*>(GetMemory(size, alignment));
}
void Reset();
// This should be the worst-case alignment for any type. This is
// good for IA-32, SPARC version 7 (the last one I know), and
// supposedly Alpha. i386 would be more time-efficient with a
// default alignment of 8, but ::operator new() uses alignment of 4,
// and an assertion will fail below after the call to MakeNewBlock()
// if you try to use a larger alignment.
#ifdef __i386__
static const int kDefaultAlignment = 4;
#else
static constexpr int kDefaultAlignment = 8;
#endif
protected:
bool SatisfyAlignment(const size_t alignment);
void MakeNewBlock(const uint32_t alignment);
void* GetMemoryFallback(const size_t size, const int align);
void* GetMemory(const size_t size, const int align) {
assert(remaining_ <= block_size_); // an invariant
if (size > 0 && size < remaining_ && align == 1) { // common case
void* result = freestart_;
freestart_ += size;
remaining_ -= size;
return result;
}
return GetMemoryFallback(size, align);
}
size_t remaining_;
private:
struct AllocatedBlock {
char* mem;
size_t size;
};
// Allocate new block of at least block_size, with the specified
// alignment.
// The returned AllocatedBlock* is valid until the next call to AllocNewBlock
// or Reset (i.e. anything that might affect overflow_blocks_).
AllocatedBlock* AllocNewBlock(const size_t block_size,
const uint32_t alignment);
const size_t block_size_;
char* freestart_; // beginning of the free space in most recent block
char* freestart_when_empty_; // beginning of the free space when we're empty
// STL vector isn't as efficient as it could be, so we use an array at first
size_t blocks_alloced_; // how many of the first_blocks_ have been alloced
AllocatedBlock first_blocks_[16]; // the length of this array is arbitrary
// if the first_blocks_ aren't enough, expand into overflow_blocks_.
std::vector<AllocatedBlock>* overflow_blocks_;
void FreeBlocks(); // Frees all except first block
Arena(const Arena&) = delete;
void operator=(const Arena&) = delete;
};
} // namespace core
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_CORE_ARENA_H_
tensorflow/core/lib/core/arena_test.cc
+121
View File
@@ -0,0 +1,121 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/lib/core/arena.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/test.h"
namespace tensorflow {
namespace core {
namespace {
// Write random data to allocated memory
static void TestMemory(void* mem, int size) {
// Check that we can memset the entire memory
memset(mem, 0xaa, size);
// Do some memory allocation to check that the arena doesn't mess up
// the internal memory allocator
char* tmp[100];
for (size_t i = 0; i < TF_ARRAYSIZE(tmp); i++) {
tmp[i] = new char[i * i + 1];
}
memset(mem, 0xcc, size);
// Free up the allocated memory;
for (size_t i = 0; i < TF_ARRAYSIZE(tmp); i++) {
delete[] tmp[i];
}
// Check that we can memset the entire memory
memset(mem, 0xee, size);
}
TEST(ArenaTest, TestBasicArena) {
Arena a(1024);
char* memory = a.Alloc(100);
ASSERT_NE(memory, nullptr);
TestMemory(memory, 100);
// Allocate again
memory = a.Alloc(100);
ASSERT_NE(memory, nullptr);
TestMemory(memory, 100);
}
TEST(ArenaTest, TestAlignment) {
Arena a(1024);
char* byte0 = a.Alloc(1);
char* alloc_aligned8 = a.AllocAligned(17, 8);
EXPECT_EQ(alloc_aligned8 - byte0, 8);
char* alloc_aligned8_b = a.AllocAligned(8, 8);
EXPECT_EQ(alloc_aligned8_b - alloc_aligned8, 24);
char* alloc_aligned8_c = a.AllocAligned(16, 8);
EXPECT_EQ(alloc_aligned8_c - alloc_aligned8_b, 8);
char* alloc_aligned8_d = a.AllocAligned(8, 1);
EXPECT_EQ(alloc_aligned8_d - alloc_aligned8_c, 16);
}
TEST(ArenaTest, TestVariousArenaSizes) {
{
Arena a(1024);
// Allocate blocksize
char* memory = a.Alloc(1024);
ASSERT_NE(memory, nullptr);
TestMemory(memory, 1024);
// Allocate another blocksize
char* memory2 = a.Alloc(1024);
ASSERT_NE(memory2, nullptr);
TestMemory(memory2, 1024);
}
// Allocate an arena and allocate two blocks
// that together exceed a block size
{
Arena a(1024);
//
char* memory = a.Alloc(768);
ASSERT_NE(memory, nullptr);
TestMemory(memory, 768);
// Allocate another blocksize
char* memory2 = a.Alloc(768);
ASSERT_NE(memory2, nullptr);
TestMemory(memory2, 768);
}
// Allocate larger than a blocksize
{
Arena a(1024);
char* memory = a.Alloc(10240);
ASSERT_NE(memory, nullptr);
TestMemory(memory, 10240);
// Allocate another blocksize
char* memory2 = a.Alloc(1234);
ASSERT_NE(memory2, nullptr);
TestMemory(memory2, 1234);
}
}
} // namespace
} // namespace core
} // namespace tensorflow
tensorflow/core/lib/core/bitmap.h
+29
View File
@@ -0,0 +1,29 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_CORE_BITMAP_H_
#define TENSORFLOW_CORE_LIB_CORE_BITMAP_H_
#include "xla/tsl/lib/core/bitmap.h"
namespace tensorflow {
namespace core {
using Bitmap = tsl::core::Bitmap;
} // namespace core
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_CORE_BITMAP_H_
tensorflow/core/lib/core/bits.h
+42
View File
@@ -0,0 +1,42 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_CORE_BITS_H_
#define TENSORFLOW_CORE_LIB_CORE_BITS_H_
#include "xla/tsl/lib/core/bits.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// NOLINTBEGIN(misc-unused-using-decls)
// Return floor(log2(n)) for positive integer n. Returns -1 iff n == 0.
using ::tsl::Log2Floor;
using ::tsl::Log2Floor64;
// Return ceiling(log2(n)) for positive integer n. Returns -1 iff n == 0.
using ::tsl::Log2Ceiling;
using ::tsl::Log2Ceiling64;
using ::tsl::NextPowerOfTwo;
using ::tsl::NextPowerOfTwo64;
// NOLINTEND(misc-unused-using-decls)
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_CORE_BITS_H_
tensorflow/core/lib/core/coding.h
+26
View File
@@ -0,0 +1,26 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Endian-neutral encoding:
// * Fixed-length numbers are encoded with least-significant byte first
// * In addition we support variable length "varint" encoding
// * Strings are encoded prefixed by their length in varint format
#ifndef TENSORFLOW_CORE_LIB_CORE_CODING_H_
#define TENSORFLOW_CORE_LIB_CORE_CODING_H_
#include "tensorflow/core/platform/coding.h" // IWYU pragma: export
#endif // TENSORFLOW_CORE_LIB_CORE_CODING_H_
tensorflow/core/lib/core/coding_test.cc
+208
View File
@@ -0,0 +1,208 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/lib/core/coding.h"
#include <vector>
#include "tensorflow/core/platform/test.h"
namespace tensorflow {
namespace core {
TEST(Coding, Fixed16) {
static const uint16_t N = 50000;
std::string s;
for (uint16_t v = 0; v < N; v++) {
char buf[sizeof(uint16_t)];
EncodeFixed16(buf, v);
s.append(buf, sizeof(buf));
}
const char* p = s.data();
for (uint16_t v = 0; v < N; v++) {
uint16_t actual = DecodeFixed16(p);
ASSERT_EQ(v, actual);
p += sizeof(uint16_t);
}
}
TEST(Coding, Fixed32) {
static const uint32_t N = 100000;
std::string s;
for (uint32_t v = 0; v < N; v++) {
char buf[sizeof(uint32_t)];
EncodeFixed32(buf, v);
s.append(buf, sizeof(buf));
}
const char* p = s.data();
for (uint32_t v = 0; v < N; v++) {
uint32_t actual = DecodeFixed32(p);
ASSERT_EQ(v, actual);
p += sizeof(uint32_t);
}
}
TEST(Coding, Fixed64) {
std::string s;
for (int power = 0; power <= 63; power++) {
uint64_t v = static_cast<uint64_t>(1) << power;
char buf[sizeof(uint64_t)];
EncodeFixed64(buf, v - 1);
s.append(buf, sizeof(buf));
EncodeFixed64(buf, v + 0);
s.append(buf, sizeof(buf));
EncodeFixed64(buf, v + 1);
s.append(buf, sizeof(buf));
}
const char* p = s.data();
for (int power = 0; power <= 63; power++) {
uint64_t v = static_cast<uint64_t>(1) << power;
uint64_t actual;
actual = DecodeFixed64(p);
ASSERT_EQ(v - 1, actual);
p += sizeof(uint64_t);
actual = DecodeFixed64(p);
ASSERT_EQ(v + 0, actual);
p += sizeof(uint64_t);
actual = DecodeFixed64(p);
ASSERT_EQ(v + 1, actual);
p += sizeof(uint64_t);
}
}
// Test that encoding routines generate little-endian encodings
TEST(Coding, EncodingOutput) {
char dst[8];
EncodeFixed16(dst, 0x0201);
ASSERT_EQ(0x01, static_cast<int>(dst[0]));
ASSERT_EQ(0x02, static_cast<int>(dst[1]));
EncodeFixed32(dst, 0x04030201);
ASSERT_EQ(0x01, static_cast<int>(dst[0]));
ASSERT_EQ(0x02, static_cast<int>(dst[1]));
ASSERT_EQ(0x03, static_cast<int>(dst[2]));
ASSERT_EQ(0x04, static_cast<int>(dst[3]));
EncodeFixed64(dst, 0x0807060504030201ull);
ASSERT_EQ(0x01, static_cast<int>(dst[0]));
ASSERT_EQ(0x02, static_cast<int>(dst[1]));
ASSERT_EQ(0x03, static_cast<int>(dst[2]));
ASSERT_EQ(0x04, static_cast<int>(dst[3]));
ASSERT_EQ(0x05, static_cast<int>(dst[4]));
ASSERT_EQ(0x06, static_cast<int>(dst[5]));
ASSERT_EQ(0x07, static_cast<int>(dst[6]));
ASSERT_EQ(0x08, static_cast<int>(dst[7]));
}
TEST(Coding, Varint32) {
std::string s;
for (uint32_t i = 0; i < (32 * 32); i++) {
uint32_t v = (i / 32) << (i % 32);
PutVarint32(&s, v);
}
const char* p = s.data();
const char* limit = p + s.size();
for (uint32_t i = 0; i < (32 * 32); i++) {
uint32_t expected = (i / 32) << (i % 32);
uint32_t actual;
p = GetVarint32Ptr(p, limit, &actual);
ASSERT_TRUE(p != nullptr);
ASSERT_EQ(expected, actual);
}
ASSERT_EQ(p, s.data() + s.size());
}
TEST(Coding, Varint64) {
// Construct the list of values to check
std::vector<uint64_t> values;
// Some special values
values.push_back(0);
values.push_back(100);
values.push_back(~static_cast<uint64_t>(0));
values.push_back(~static_cast<uint64_t>(0) - 1);
for (uint32_t k = 0; k < 64; k++) {
// Test values near powers of two
const uint64_t power = 1ull << k;
values.push_back(power);
values.push_back(power - 1);
values.push_back(power + 1);
}
std::string s;
for (size_t i = 0; i < values.size(); i++) {
PutVarint64(&s, values[i]);
}
const char* p = s.data();
const char* limit = p + s.size();
for (size_t i = 0; i < values.size(); i++) {
ASSERT_TRUE(p < limit);
uint64_t actual;
p = GetVarint64Ptr(p, limit, &actual);
ASSERT_TRUE(p != nullptr);
ASSERT_EQ(values[i], actual);
}
ASSERT_EQ(p, limit);
}
TEST(Coding, Varint32Overflow) {
uint32_t result;
std::string input("\x81\x82\x83\x84\x85\x11");
ASSERT_TRUE(GetVarint32Ptr(input.data(), input.data() + input.size(),
&result) == nullptr);
}
TEST(Coding, Varint32Truncation) {
uint32_t large_value = (1u << 31) + 100;
std::string s;
PutVarint32(&s, large_value);
uint32_t result;
for (size_t len = 0; len < s.size() - 1; len++) {
ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + len, &result) == nullptr);
}
ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + s.size(), &result) !=
nullptr);
ASSERT_EQ(large_value, result);
}
TEST(Coding, Varint64Overflow) {
uint64_t result;
std::string input("\x81\x82\x83\x84\x85\x81\x82\x83\x84\x85\x11");
ASSERT_TRUE(GetVarint64Ptr(input.data(), input.data() + input.size(),
&result) == nullptr);
}
TEST(Coding, Varint64Truncation) {
uint64_t large_value = (1ull << 63) + 100ull;
std::string s;
PutVarint64(&s, large_value);
uint64_t result;
for (size_t len = 0; len < s.size() - 1; len++) {
ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + len, &result) == nullptr);
}
ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + s.size(), &result) !=
nullptr);
ASSERT_EQ(large_value, result);
}
} // namespace core
} // namespace tensorflow
tensorflow/core/lib/core/error_codes.proto
+18
View File
@@ -0,0 +1,18 @@
// Copyright 2026 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ==============================================================================
syntax = "proto3";
import public "xla/tsl/protobuf/error_codes.proto";
tensorflow/core/lib/core/errors.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_CORE_ERRORS_H_
#define TENSORFLOW_CORE_LIB_CORE_ERRORS_H_
#include "tensorflow/core/platform/errors.h" // IWYU pragma: export
#endif // TENSORFLOW_CORE_LIB_CORE_ERRORS_H_
tensorflow/core/lib/core/notification.h
+19
View File
@@ -0,0 +1,19 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_CORE_NOTIFICATION_H_
#define TENSORFLOW_CORE_LIB_CORE_NOTIFICATION_H_
#endif // TENSORFLOW_CORE_LIB_CORE_NOTIFICATION_H_
tensorflow/core/lib/core/raw_coding.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_CORE_RAW_CODING_H_
#define TENSORFLOW_CORE_LIB_CORE_RAW_CODING_H_
#include "tensorflow/core/platform/raw_coding.h" // IWYU pragma: export
#endif // TENSORFLOW_CORE_LIB_CORE_RAW_CODING_H_
tensorflow/core/lib/core/refcount.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_CORE_REFCOUNT_H_
#define TENSORFLOW_CORE_LIB_CORE_REFCOUNT_H_
#include "tensorflow/core/platform/refcount.h" // IWYU pragma: export
#endif // TENSORFLOW_CORE_LIB_CORE_REFCOUNT_H_
tensorflow/core/lib/core/status.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_CORE_STATUS_H_
#define TENSORFLOW_CORE_LIB_CORE_STATUS_H_
#include "tensorflow/core/platform/status.h" // IWYU pragma: export
#endif // TENSORFLOW_CORE_LIB_CORE_STATUS_H_
tensorflow/core/lib/core/status_test.cc
+208
View File
@@ -0,0 +1,208 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/lib/core/status.h"
#include "absl/strings/match.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status_test_util.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/platform/test_benchmark.h"
namespace tensorflow {
TEST(Status, OK) {
EXPECT_EQ(absl::OkStatus().code(), error::OK);
EXPECT_EQ(absl::OkStatus().message(), "");
TF_EXPECT_OK(absl::OkStatus());
TF_ASSERT_OK(absl::OkStatus());
EXPECT_EQ(absl::OkStatus(), absl::Status());
absl::Status s;
EXPECT_TRUE(s.ok());
}
TEST(DeathStatus, CheckOK) {
absl::Status status(absl::InvalidArgumentError("Invalid"));
ASSERT_DEATH(TF_CHECK_OK(status), "Invalid");
}
TEST(Status, Set) {
absl::Status status;
status = absl::Status(absl::StatusCode::kCancelled, "Error message");
EXPECT_EQ(status.code(), absl::StatusCode::kCancelled);
EXPECT_EQ(status.message(), "Error message");
}
TEST(Status, Copy) {
absl::Status a(absl::InvalidArgumentError("Invalid"));
absl::Status b(a);
ASSERT_EQ(a.ToString(), b.ToString());
}
TEST(Status, Assign) {
absl::Status a(absl::InvalidArgumentError("Invalid"));
absl::Status b;
b = a;
ASSERT_EQ(a.ToString(), b.ToString());
}
TEST(Status, Move) {
absl::Status a(absl::InvalidArgumentError("Invalid"));
absl::Status b(std::move(a));
ASSERT_EQ("INVALID_ARGUMENT: Invalid", b.ToString());
}
TEST(Status, MoveAssign) {
absl::Status a(absl::InvalidArgumentError("Invalid"));
absl::Status b;
b = std::move(a);
ASSERT_EQ("INVALID_ARGUMENT: Invalid", b.ToString());
}
TEST(Status, Update) {
absl::Status s;
s.Update(absl::OkStatus());
ASSERT_TRUE(s.ok());
absl::Status a(absl::InvalidArgumentError("Invalid"));
s.Update(a);
ASSERT_EQ(s.ToString(), a.ToString());
absl::Status b(absl::InternalError("Internal"));
s.Update(b);
ASSERT_EQ(s.ToString(), a.ToString());
s.Update(absl::OkStatus());
ASSERT_EQ(s.ToString(), a.ToString());
ASSERT_FALSE(s.ok());
}
TEST(Status, EqualsOK) { ASSERT_EQ(absl::OkStatus(), absl::Status()); }
TEST(Status, EqualsSame) {
absl::Status a(absl::InvalidArgumentError("Invalid"));
absl::Status b(absl::InvalidArgumentError("Invalid"));
ASSERT_EQ(a, b);
}
TEST(Status, EqualsCopy) {
const absl::Status a(absl::InvalidArgumentError("Invalid"));
const absl::Status b = a;
ASSERT_EQ(a, b);
}
TEST(Status, EqualsDifferentCode) {
const absl::Status a(absl::InvalidArgumentError("message"));
const absl::Status b(absl::InternalError("message"));
ASSERT_NE(a, b);
}
TEST(Status, EqualsDifferentMessage) {
const absl::Status a(absl::InvalidArgumentError("message"));
const absl::Status b(absl::InvalidArgumentError("another"));
ASSERT_NE(a, b);
}
TEST(StatusGroup, OKStatusGroup) {
StatusGroup c;
c.Update(absl::OkStatus());
c.Update(absl::OkStatus());
ASSERT_EQ(c.as_summary_status(), absl::OkStatus());
ASSERT_EQ(c.as_concatenated_status(), absl::OkStatus());
}
TEST(StatusGroup, AggregateWithSingleErrorStatus) {
StatusGroup c;
const absl::Status internal(absl::InternalError("Original error."));
c.Update(internal);
ASSERT_EQ(c.as_summary_status(), internal);
absl::Status concat_status = c.as_concatenated_status();
ASSERT_EQ(concat_status.code(), internal.code());
ASSERT_TRUE(absl::StrContains(concat_status.message(), internal.message()));
// Add derived error status
const absl::Status derived =
StatusGroup::MakeDerived(absl::InternalError("Derived error."));
c.Update(derived);
ASSERT_EQ(c.as_summary_status(), internal);
concat_status = c.as_concatenated_status();
ASSERT_EQ(concat_status.code(), internal.code());
ASSERT_TRUE(absl::StrContains(concat_status.message(), internal.message()));
}
TEST(StatusGroup, AggregateWithMultipleErrorStatus) {
StatusGroup c;
const absl::Status internal(absl::InternalError("Original error."));
const absl::Status cancelled(
absl::CancelledError("Cancelled after 10 steps."));
const absl::Status aborted(absl::AbortedError("Aborted after 10 steps."));
c.Update(internal);
c.Update(cancelled);
c.Update(aborted);
absl::Status summary = c.as_summary_status();
ASSERT_EQ(summary.code(), internal.code());
ASSERT_TRUE(absl::StrContains(summary.message(), internal.message()));
ASSERT_TRUE(absl::StrContains(summary.message(), cancelled.message()));
ASSERT_TRUE(absl::StrContains(summary.message(), aborted.message()));
absl::Status concat_status = c.as_concatenated_status();
ASSERT_EQ(concat_status.code(), internal.code());
ASSERT_TRUE(absl::StrContains(concat_status.message(), internal.message()));
ASSERT_TRUE(absl::StrContains(concat_status.message(), cancelled.message()));
ASSERT_TRUE(absl::StrContains(concat_status.message(), aborted.message()));
}
TEST(Status, InvalidPayloadGetsIgnored) {
absl::Status s = absl::Status();
s.SetPayload("Invalid", absl::Cord("Invalid Val"));
ASSERT_FALSE(s.GetPayload("Invalid").has_value());
bool is_err_erased = s.ErasePayload("Invalid");
ASSERT_EQ(is_err_erased, false);
}
TEST(Status, SetPayloadSetsOrUpdatesIt) {
absl::Status s(absl::StatusCode::kInternal, "Error message");
s.SetPayload("Error key", absl::Cord("Original"));
ASSERT_EQ(s.GetPayload("Error key"), absl::Cord("Original"));
s.SetPayload("Error key", absl::Cord("Updated"));
ASSERT_EQ(s.GetPayload("Error key"), absl::Cord("Updated"));
}
TEST(Status, ErasePayloadRemovesIt) {
absl::Status s(absl::StatusCode::kInternal, "Error message");
s.SetPayload("Error key", absl::Cord("Original"));
bool is_err_erased = s.ErasePayload("Error key");
ASSERT_EQ(is_err_erased, true);
is_err_erased = s.ErasePayload("Error key");
ASSERT_EQ(is_err_erased, false);
ASSERT_FALSE(s.GetPayload("Error key").has_value());
}
static void BM_TF_CHECK_OK(::testing::benchmark::State& state) {
absl::Status s = (state.max_iterations < 0)
? absl::InvalidArgumentError("Invalid")
: absl::OkStatus();
for (auto i : state) {
TF_CHECK_OK(s);
}
}
BENCHMARK(BM_TF_CHECK_OK);
} // namespace tensorflow
tensorflow/core/lib/core/status_test_util.h
+22
View File
@@ -0,0 +1,22 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_CORE_STATUS_TEST_UTIL_H_
#define TENSORFLOW_CORE_LIB_CORE_STATUS_TEST_UTIL_H_
#include "xla/tsl/lib/core/status_test_util.h"
#include "tensorflow/core/platform/test.h"
#endif // TENSORFLOW_CORE_LIB_CORE_STATUS_TEST_UTIL_H_
tensorflow/core/lib/core/stringpiece.h
+31
View File
@@ -0,0 +1,31 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// StringPiece is a simple structure containing a pointer into some external
// storage and a size. The user of a StringPiece must ensure that the slice
// is not used after the corresponding external storage has been
// deallocated.
//
// Multiple threads can invoke const methods on a StringPiece without
// external synchronization, but if any of the threads may call a
// non-const method, all threads accessing the same StringPiece must use
// external synchronization.
#ifndef TENSORFLOW_CORE_LIB_CORE_STRINGPIECE_H_
#define TENSORFLOW_CORE_LIB_CORE_STRINGPIECE_H_
#include "tensorflow/core/platform/stringpiece.h" // IWYU pragma: export
#endif // TENSORFLOW_CORE_LIB_CORE_STRINGPIECE_H_
tensorflow/core/lib/core/threadpool.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_CORE_THREADPOOL_H_
#define TENSORFLOW_CORE_LIB_CORE_THREADPOOL_H_
#include "tensorflow/core/platform/threadpool.h" // IWYU pragma: export
#endif // TENSORFLOW_CORE_LIB_CORE_THREADPOOL_H_
tensorflow/core/lib/core/threadpool_interface.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_CORE_THREADPOOL_INTERFACE_H_
#define TENSORFLOW_CORE_LIB_CORE_THREADPOOL_INTERFACE_H_
#include "tensorflow/core/platform/threadpool_interface.h" // IWYU pragma: export
#endif // TENSORFLOW_CORE_LIB_CORE_THREADPOOL_INTERFACE_H_
tensorflow/core/lib/core/threadpool_options.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_CORE_THREADPOOL_OPTIONS_H_
#define TENSORFLOW_CORE_LIB_CORE_THREADPOOL_OPTIONS_H_
#include "tensorflow/core/platform/threadpool_options.h" // IWYU pragma: export
#endif // TENSORFLOW_CORE_LIB_CORE_THREADPOOL_OPTIONS_H_
tensorflow/core/lib/core/threadpool_test.cc
+466
View File
@@ -0,0 +1,466 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/lib/core/threadpool.h"
#include <atomic>
#include <cstdint>
#include <functional>
#include <optional>
#include "absl/synchronization/barrier.h"
#include "absl/synchronization/mutex.h"
#include "absl/time/time.h"
#include "tensorflow/core/platform/context.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/platform/test_benchmark.h"
#include "tsl/platform/platform.h" // IWYU pragma: keep
namespace tensorflow {
namespace thread {
static const int kNumThreads = 30;
TEST(ThreadPool, Empty) {
for (int num_threads = 1; num_threads < kNumThreads; num_threads++) {
ThreadPool pool(Env::Default(), "test", num_threads);
}
}
TEST(ThreadPool, DoWork) {
Context outer_context(ContextKind::kThread);
for (int num_threads = 1; num_threads < kNumThreads; num_threads++) {
const int kWorkItems = 15;
std::atomic<bool> work[kWorkItems];
for (int i = 0; i < kWorkItems; i++) {
work[i] = false;
}
{
ThreadPool pool(Env::Default(), "test", num_threads);
for (int i = 0; i < kWorkItems; i++) {
pool.Schedule([&outer_context, &work, i]() {
Context inner_context(ContextKind::kThread);
ASSERT_EQ(outer_context, inner_context);
ASSERT_FALSE(work[i].exchange(true));
});
}
}
for (int i = 0; i < kWorkItems; i++) {
ASSERT_TRUE(work[i]);
}
}
}
void RunWithFixedBlockSize(int64_t block_size, int64_t total,
ThreadPool* threads) {
mutex mu;
int64_t num_shards = 0;
int64_t num_done_work = 0;
std::vector<std::atomic<bool>> work(total);
for (int i = 0; i < total; i++) {
work[i] = false;
}
threads->ParallelFor(
total,
ThreadPool::SchedulingParams(
ThreadPool::SchedulingStrategy::kFixedBlockSize /* strategy */,
std::nullopt /* cost_per_unit */, block_size /* block_size */),
[=, &mu, &num_shards, &num_done_work, &work](int64_t start, int64_t end) {
VLOG(1) << "Shard [" << start << "," << end << ")";
EXPECT_GE(start, 0);
EXPECT_LE(end, total);
mutex_lock l(mu);
++num_shards;
for (; start < end; ++start) {
EXPECT_FALSE(work[start].exchange(true)); // No duplicate
++num_done_work;
}
});
EXPECT_EQ(num_done_work, total);
for (int i = 0; i < total; i++) {
ASSERT_TRUE(work[i]);
}
const int64_t num_workers = (total + block_size - 1) / block_size;
if (num_workers < threads->NumThreads()) {
// If the intention is to limit the parallelism explicitly, we'd
// better honor it. Ideally, even if per_thread_max_parallelism >
// num_workers, we should expect that Shard() implementation do
// not over-shard. Unfortunately, ThreadPoolDevice::parallelFor
// tends to over-shard.
EXPECT_LE(num_shards, 1 + num_workers);
}
}
// Adapted from work_sharder_test.cc
TEST(ThreadPoolTest, ParallelForFixedBlockSizeScheduling) {
ThreadPool threads(Env::Default(), "test", 16);
for (auto block_size : {1, 7, 10, 64, 100, 256, 1000, 9999}) {
for (auto diff : {0, 1, 11, 102, 1003, 10005, 1000007}) {
const int64_t total = block_size + diff;
RunWithFixedBlockSize(block_size, total, &threads);
}
}
}
void RunWithFixedBlockSizeTransformRangeConcurrently(int64_t block_size,
int64_t total,
ThreadPool* threads) {
mutex mu;
int64_t num_shards = 0;
int64_t num_done_work = 0;
std::vector<std::atomic<bool>> work(total);
for (int i = 0; i < total; i++) {
work[i] = false;
}
threads->TransformRangeConcurrently(
block_size, total,
[=, &mu, &num_shards, &num_done_work, &work](int64_t start, int64_t end) {
VLOG(1) << "Shard [" << start << "," << end << ")";
EXPECT_GE(start, 0);
EXPECT_LE(end, total);
mutex_lock l(mu);
++num_shards;
for (; start < end; ++start) {
EXPECT_FALSE(work[start].exchange(true)); // No duplicate
++num_done_work;
}
});
EXPECT_EQ(num_done_work, total);
for (int i = 0; i < total; i++) {
ASSERT_TRUE(work[i]);
}
const int64_t num_workers = (total + block_size - 1) / block_size;
if (num_workers < threads->NumThreads()) {
// If the intention is to limit the parallelism explicitly, we'd
// better honor it. Ideally, even if per_thread_max_parallelism >
// num_workers, we should expect that Shard() implementation do
// not over-shard. Unfortunately, ThreadPoolDevice::parallelFor
// tends to over-shard.
EXPECT_LE(num_shards, 1 + num_workers);
}
}
// Adapted from work_sharder_test.cc
TEST(ThreadPoolTest, TransformRangeConcurrently) {
ThreadPool threads(Env::Default(), "test", 16);
for (auto block_size : {1, 7, 10, 64, 100, 256, 1000, 9999}) {
for (auto diff : {0, 1, 11, 102, 1003, 10005, 1000007}) {
const int64_t total = block_size + diff;
RunWithFixedBlockSizeTransformRangeConcurrently(block_size, total,
&threads);
}
}
}
TEST(ThreadPoolTest, NumShardsUsedByFixedBlockSizeScheduling) {
ThreadPool threads(Env::Default(), "test", 16);
EXPECT_EQ(1, threads.NumShardsUsedByFixedBlockSizeScheduling(
3 /* total */, 3 /* block_size */));
EXPECT_EQ(2, threads.NumShardsUsedByFixedBlockSizeScheduling(
4 /* total */, 3 /* block_size */));
EXPECT_EQ(2, threads.NumShardsUsedByFixedBlockSizeScheduling(
5 /* total */, 3 /* block_size */));
EXPECT_EQ(2, threads.NumShardsUsedByFixedBlockSizeScheduling(
6 /* total */, 3 /* block_size */));
EXPECT_EQ(3, threads.NumShardsUsedByFixedBlockSizeScheduling(
7 /* total */, 3 /* block_size */));
EXPECT_EQ(7, threads.NumShardsUsedByFixedBlockSizeScheduling(
7 /* total */, 1 /* block_size */));
EXPECT_EQ(1, threads.NumShardsUsedByFixedBlockSizeScheduling(
7 /* total */, 0 /* block_size */));
}
TEST(ThreadPoolTest, NumShardsUsedByTransformRangeConcurrently) {
ThreadPool threads(Env::Default(), "test", 16);
EXPECT_EQ(1, threads.NumShardsUsedByTransformRangeConcurrently(
3 /* block_size */, 3 /* total */));
EXPECT_EQ(2, threads.NumShardsUsedByTransformRangeConcurrently(
3 /* block_size */, 4 /* total */));
EXPECT_EQ(2, threads.NumShardsUsedByTransformRangeConcurrently(
3 /* block_size */, 5 /* total */));
EXPECT_EQ(2, threads.NumShardsUsedByTransformRangeConcurrently(
3 /* block_size */, 6 /* total */));
EXPECT_EQ(3, threads.NumShardsUsedByTransformRangeConcurrently(
3 /* block_size */, 7 /* total */));
EXPECT_EQ(7, threads.NumShardsUsedByTransformRangeConcurrently(
1 /* block_size */, 7 /* total */));
EXPECT_EQ(1, threads.NumShardsUsedByTransformRangeConcurrently(
0 /* block_size */, 7 /* total */));
}
void RunFixedBlockSizeShardingWithWorkerId(int64_t block_size, int64_t total,
ThreadPool* threads) {
mutex mu;
int64_t num_done_work = 0;
std::vector<std::atomic<bool>> work(total);
for (int i = 0; i < total; i++) {
work[i] = false;
}
const int64_t num_threads = threads->NumThreads();
std::vector<std::atomic<bool>> threads_running(num_threads + 1);
for (int i = 0; i < num_threads + 1; i++) {
threads_running[i] = false;
}
threads->ParallelForWithWorkerId(
total,
ThreadPool::SchedulingParams(
ThreadPool::SchedulingStrategy::kFixedBlockSize /* strategy */,
std::nullopt /* cost_per_unit */, block_size /* block_size */),
[=, &mu, &num_done_work, &work, &threads_running](int64_t start,
int64_t end, int id) {
VLOG(1) << "Shard [" << start << "," << end << ")";
EXPECT_GE(start, 0);
EXPECT_LE(end, total);
// Store true for the current thread, and assert that another thread
// is not running with the same id.
EXPECT_GE(id, 0);
EXPECT_LE(id, num_threads);
EXPECT_FALSE(threads_running[id].exchange(true));
mutex_lock l(mu);
for (; start < end; ++start) {
EXPECT_FALSE(work[start].exchange(true)); // No duplicate
++num_done_work;
}
EXPECT_TRUE(threads_running[id].exchange(false));
});
EXPECT_EQ(num_done_work, total);
for (int i = 0; i < total; i++) {
EXPECT_TRUE(work[i]);
}
}
TEST(ThreadPoolTest, ParallelForFixedBlockSizeSchedulingWithWorkerId) {
for (int32_t num_threads : {1, 2, 3, 9, 16, 31}) {
ThreadPool threads(Env::Default(), "test", num_threads);
for (int64_t block_size : {1, 7, 10, 64, 100, 256, 1000}) {
for (int64_t diff : {0, 1, 11, 102, 1003}) {
const int64_t total = block_size + diff;
RunFixedBlockSizeShardingWithWorkerId(block_size, total, &threads);
}
}
}
}
TEST(ThreadPool, ParallelFor) {
Context outer_context(ContextKind::kThread);
// Make ParallelFor use as many threads as possible.
int64_t kHugeCost = 1 << 30;
for (int num_threads = 1; num_threads < kNumThreads; num_threads++) {
const int kWorkItems = 15;
std::atomic<bool> work[kWorkItems];
ThreadPool pool(Env::Default(), "test", num_threads);
for (int i = 0; i < kWorkItems; i++) {
work[i] = false;
}
pool.ParallelFor(kWorkItems,
ThreadPool::SchedulingParams::Adaptive(kHugeCost),
[&outer_context, &work](int64_t begin, int64_t end) {
Context inner_context(ContextKind::kThread);
ASSERT_EQ(outer_context, inner_context);
for (int64_t i = begin; i < end; ++i) {
ASSERT_FALSE(work[i].exchange(true));
}
});
for (int i = 0; i < kWorkItems; i++) {
ASSERT_TRUE(work[i]);
}
}
}
TEST(ThreadPool, ParallelForWithAdaptiveSchedulingStrategy) {
Context outer_context(ContextKind::kThread);
// Make ParallelFor use as many threads as possible.
int64_t kHugeCost = 1 << 30;
for (int num_threads = 1; num_threads < kNumThreads; num_threads++) {
const int kWorkItems = 15;
std::atomic<bool> work[kWorkItems];
ThreadPool pool(Env::Default(), "test", num_threads);
for (int i = 0; i < kWorkItems; i++) {
work[i] = false;
}
pool.ParallelFor(
kWorkItems,
ThreadPool::SchedulingParams(
ThreadPool::SchedulingStrategy::kAdaptive /* strategy */,
kHugeCost /* cost_per_unit */, std::nullopt /* block_size */),
[&outer_context, &work](int64_t begin, int64_t end) {
Context inner_context(ContextKind::kThread);
ASSERT_EQ(outer_context, inner_context);
for (int64_t i = begin; i < end; ++i) {
ASSERT_FALSE(work[i].exchange(true));
}
});
for (int i = 0; i < kWorkItems; i++) {
ASSERT_TRUE(work[i]);
}
}
}
TEST(ThreadPool, ParallelForWithWorkerId) {
// Make ParallelForWithWorkerId use as many threads as possible.
int64_t kHugeCost = 1 << 30;
for (int num_threads = 1; num_threads < kNumThreads; num_threads++) {
const int kWorkItems = 15;
std::atomic<bool> work[kWorkItems];
ThreadPool pool(Env::Default(), "test", num_threads);
for (int i = 0; i < kWorkItems; i++) {
work[i] = false;
}
std::atomic<bool> threads_running[kNumThreads + 1];
for (int i = 0; i < num_threads + 1; i++) {
threads_running[i] = false;
}
pool.ParallelForWithWorkerId(
kWorkItems, ThreadPool::SchedulingParams::Adaptive(kHugeCost),
[&threads_running, &work](int64_t begin, int64_t end, int64_t id) {
// Store true for the current thread, and assert that another thread
// is not running with the same id.
ASSERT_LE(0, id);
ASSERT_LE(id, kNumThreads);
ASSERT_FALSE(threads_running[id].exchange(true));
for (int64_t i = begin; i < end; ++i) {
ASSERT_FALSE(work[i].exchange(true));
}
ASSERT_TRUE(threads_running[id].exchange(false));
threads_running[id] = false;
});
for (int i = 0; i < kWorkItems; i++) {
ASSERT_TRUE(work[i]);
}
for (int i = 0; i < num_threads + 1; i++) {
ASSERT_FALSE(threads_running[i]);
}
}
}
TEST(ThreadPool, Parallelism) {
// Test that if we have N threads and schedule N tasks,
// all tasks will be scheduled at the same time.
// Failure mode for this test will be timeouts.
ThreadPool pool(Env::Default(), "test", kNumThreads);
for (int iter = 0; iter < 2000; iter++) {
absl::Barrier barrier(kNumThreads);
// Expect each loop finishes less than 1s or much less. The semantic of
// counter, mutex and done here is the same as absl::BlockingCounter except
// that it waits for the condition with timeout.
std::atomic<int> counter(kNumThreads);
absl::Mutex mutex;
bool done = false;
for (int t = 0; t < kNumThreads; ++t) {
pool.Schedule([&]() {
barrier.Block();
if (--counter <= 0) {
absl::MutexLock lock(mutex);
done = true;
}
});
}
absl::MutexLock lock(mutex);
absl::Condition cond(+[](bool* done) { return *done; }, &done);
EXPECT_TRUE(mutex.AwaitWithTimeout(cond, absl::Seconds(1)));
}
}
static void BM_Sequential(::testing::benchmark::State& state) {
for (auto s : state) {
state.PauseTiming();
ThreadPool pool(Env::Default(), "test", kNumThreads);
// Decrement count sequentially until 0.
int count = state.range(0);
mutex done_lock;
bool done_flag = false;
std::function<void()> work = [&pool, &count, &done_lock, &done_flag,
&work]() {
if (count--) {
pool.Schedule(work);
} else {
mutex_lock l(done_lock);
done_flag = true;
}
};
state.ResumeTiming();
work();
mutex_lock l(done_lock);
done_lock.Await(Condition(&done_flag));
}
}
BENCHMARK(BM_Sequential)->Arg(200)->Arg(300);
static void BM_Parallel(::testing::benchmark::State& state) {
ThreadPool pool(Env::Default(), "test", kNumThreads);
// Decrement count concurrently until 0.
std::atomic_int_fast32_t count(state.max_iterations);
mutex done_lock;
bool done_flag = false;
for (auto s : state) {
pool.Schedule([&count, &done_lock, &done_flag]() {
if (count.fetch_sub(1) == 1) {
mutex_lock l(done_lock);
done_flag = true;
}
});
}
mutex_lock l(done_lock);
done_lock.Await(Condition(&done_flag));
}
BENCHMARK(BM_Parallel);
static void BM_ParallelFor(::testing::benchmark::State& state) {
int total = state.range(0);
int cost_per_unit = state.range(1);
ThreadPool pool(Env::Default(), "test", kNumThreads);
// Decrement count concurrently until 0.
std::atomic_int_fast32_t count(state.max_iterations);
mutex done_lock;
bool done_flag = false;
for (auto s : state) {
pool.ParallelFor(
total, cost_per_unit,
[&count, &done_lock, &done_flag](int64_t begin, int64_t end) {
for (int64_t i = begin; i < end; ++i) {
if (count.fetch_sub(1) == 1) {
mutex_lock l(done_lock);
done_flag = true;
}
}
});
mutex_lock l(done_lock);
done_lock.Await(Condition(&done_flag));
}
}
BENCHMARK(BM_ParallelFor)
->ArgPair(1 << 10, 1)
->ArgPair(1 << 20, 1)
->ArgPair(1 << 10, 1 << 10)
->ArgPair(1 << 20, 1 << 10)
->ArgPair(1 << 10, 1 << 20)
->ArgPair(1 << 20, 1 << 20)
->ArgPair(1 << 10, 1 << 30)
->ArgPair(1 << 20, 1 << 30);
} // namespace thread
} // namespace tensorflow
tensorflow/core/lib/db/BUILD
+57
View File
@@ -0,0 +1,57 @@
# Description:
# Libraries for storing tensors in SQL databases.
load("//tensorflow:tensorflow.bzl", "tf_cc_test", "tf_copts")
load(
"//tensorflow/core/platform:rules_cc.bzl",
"cc_library",
)
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow:internal"],
licenses = ["notice"],
)
cc_library(
name = "sqlite",
srcs = ["sqlite.cc"],
hdrs = ["sqlite.h"],
copts = tf_copts(),
deps = [
":snapfn",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"@com_google_absl//absl/log",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/strings:string_view",
"@org_sqlite",
],
)
cc_library(
name = "snapfn",
srcs = ["snapfn.cc"],
copts = tf_copts() + ["-DSQLITE_OMIT_LOAD_EXTENSION"],
linkstatic = 1,
deps = [
"@org_sqlite",
"@snappy",
],
alwayslink = 1,
)
tf_cc_test(
name = "sqlite_test",
size = "small",
srcs = ["sqlite_test.cc"],
deps = [
":sqlite",
"//tensorflow/core:lib",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
],
)
tensorflow/core/lib/db/snapfn.cc
+253
View File
@@ -0,0 +1,253 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
/// \brief SQLite extension for Snappy compression
///
/// Snappy a compression library that trades ratio for speed, almost going a
/// tenth as fast as memcpy().
///
/// FUNCTIONS
///
/// - snap(value: BLOB|TEXT) -> BLOB
/// - snap(value: NULL|INT|REAL) -> value
///
/// Applies Snappy compression. If value is TEXT or BLOB, then it is
/// compressed and a BLOB is returned with a byte prepended to indicate the
/// original type. Other types are returned as-is.
///
/// - unsnap(value: BLOB) -> TEXT|BLOB
/// - unsnap(value: TEXT) -> SQLITE_MISMATCH
/// - unsnap(value: NULL|INT|REAL) -> value
///
/// Decompresses value created by snap(). If value is empty, then an empty
/// blob is returned. Otherwise the original type is restored from the first
/// byte and the remaining ones are decompressed. TEXT is not allowed as an
/// input type. Remaining types are returned as-is.
///
/// PERFORMANCE CONSIDERATIONS
///
/// These functions are deterministic. This means SQLite ≥3.8.3 will factor
/// them out of inner loops when constant arguments are provided. In SQLite
/// ≥3.15.0 they can be used in the WHERE clause of partial indexes. Currently
/// there is no support for common sub-expression elimination.
///
/// SQLite environments that aren't universally UTF8 will work, but should
/// encounter superfluous charset transcodings; as this implementation encodes
/// only UTF8 TEXT for the sake of simplicity. Contributions are welcome that
/// register multiple sister functions for the various charsets, which use the
/// higher order bits of the type byte to indicate encoding.
///
/// SUPPORT MATRIX
///
/// - 3.20.0 (2016-05-18) What FOSS TensorFlow uses
/// - 3.13.0 (2016-05-18) What Google uses c. 2017-12
/// - 3.8.2 (2013-12-06) Used by Ubuntu 14.04
///
/// MANUAL COMPILATION
///
/// $ sudo apt-get install libsqlite3-dev libsnappy-dev
/// $ c++ -shared --std=c++11 -o libsnapfn.so -fPIC snapfn.cc -lsnappy
///
/// $ sqlite3
/// sqlite> .load libsnapfn.so
/// sqlite> select hex(snap('aaaaaaaaaaaaaaaaa'));
/// 031100613E0100
/// sqlite> select unsnap(x'031100613E0100');
/// aaaaaaaaaaaaaaaaa
///
/// $ python
/// >>> import sqlite3
/// >>> db = sqlite3.connect(':memory:')
/// >>> db.enable_load_extension(True)
/// >>> db.execute('select load_extension("libsnapfn.so")')
/// >>> db.enable_load_extension(False)
/// >>> db.execute('select hex(snap("aaaaaaaaaaaaaaaaa"))').fetchone()[0]
/// u'031100613E0100'
#include "sqlite3ext.h"
#include "snappy.h"
SQLITE_EXTENSION_INIT1
static void snap(sqlite3_context* ctx, int /*argc*/, sqlite3_value** argv) {
const char* data;
int type = sqlite3_value_type(argv[0]);
switch (type) {
case SQLITE_NULL:
return;
case SQLITE_INTEGER:
sqlite3_result_int64(ctx, sqlite3_value_int64(argv[0]));
return;
case SQLITE_FLOAT:
sqlite3_result_double(ctx, sqlite3_value_double(argv[0]));
return;
case SQLITE_BLOB:
data = reinterpret_cast<const char*>(sqlite3_value_blob(argv[0]));
break;
case SQLITE_TEXT:
data = reinterpret_cast<const char*>(sqlite3_value_text(argv[0]));
break;
default:
sqlite3_result_error(ctx, "snap() invalid type", -1);
sqlite3_result_error_code(ctx, SQLITE_MISMATCH);
return;
}
int size = sqlite3_value_bytes(argv[0]);
if (size <= 0) {
char result[] = {static_cast<char>(type)};
sqlite3_result_blob(ctx, result, sizeof(result), SQLITE_TRANSIENT);
return;
}
size_t output_size =
snappy::MaxCompressedLength(static_cast<size_t>(size)) + 1;
if (output_size >
static_cast<size_t>(sqlite3_limit(sqlite3_context_db_handle(ctx),
SQLITE_LIMIT_LENGTH, -1))) {
sqlite3_result_error_toobig(ctx);
return;
}
auto output =
static_cast<char*>(sqlite3_malloc(static_cast<int>(output_size)));
if (output == nullptr) {
sqlite3_result_error_nomem(ctx);
return;
}
*output++ = static_cast<char>(type), --output_size;
snappy::RawCompress(data, static_cast<size_t>(size), output, &output_size);
sqlite3_result_blob(ctx, output - 1, static_cast<int>(output_size + 1),
sqlite3_free);
}
static void unsnap(sqlite3_context* ctx, int /*argc*/, sqlite3_value** argv) {
int type = sqlite3_value_type(argv[0]);
switch (type) {
case SQLITE_NULL:
return;
case SQLITE_INTEGER:
sqlite3_result_int64(ctx, sqlite3_value_int64(argv[0]));
return;
case SQLITE_FLOAT:
sqlite3_result_double(ctx, sqlite3_value_double(argv[0]));
return;
case SQLITE_BLOB:
break;
default:
sqlite3_result_error(ctx, "unsnap() invalid type", -1);
sqlite3_result_error_code(ctx, SQLITE_MISMATCH);
return;
}
int size = sqlite3_value_bytes(argv[0]);
auto blob = reinterpret_cast<const char*>(sqlite3_value_blob(argv[0]));
if (size <= 0) {
sqlite3_result_zeroblob(ctx, 0);
return;
}
type = static_cast<int>(*blob++), --size;
if (type != SQLITE_BLOB && type != SQLITE_TEXT) {
sqlite3_result_error(ctx, "unsnap() first byte is invalid type", -1);
sqlite3_result_error_code(ctx, SQLITE_CORRUPT);
return;
}
if (size == 0) {
if (type == SQLITE_TEXT) {
sqlite3_result_text(ctx, "", 0, SQLITE_STATIC);
} else {
sqlite3_result_zeroblob(ctx, 0);
}
return;
}
size_t output_size;
if (!snappy::GetUncompressedLength(blob, static_cast<size_t>(size),
&output_size)) {
sqlite3_result_error(ctx, "snappy parse error", -1);
sqlite3_result_error_code(ctx, SQLITE_CORRUPT);
return;
}
if (output_size >
static_cast<size_t>(sqlite3_limit(sqlite3_context_db_handle(ctx),
SQLITE_LIMIT_LENGTH, -1))) {
sqlite3_result_error_toobig(ctx);
return;
}
auto output =
static_cast<char*>(sqlite3_malloc(static_cast<int>(output_size)));
if (output == nullptr) {
sqlite3_result_error_nomem(ctx);
return;
}
if (!snappy::RawUncompress(blob, static_cast<size_t>(size), output)) {
sqlite3_result_error(ctx, "snappy message corruption", -1);
sqlite3_result_error_code(ctx, SQLITE_CORRUPT);
sqlite3_free(output);
return;
}
if (type == SQLITE_TEXT) {
sqlite3_result_text(ctx, output, static_cast<int>(output_size),
sqlite3_free);
} else {
sqlite3_result_blob(ctx, output, static_cast<int>(output_size),
sqlite3_free);
}
}
extern "C" {
#ifndef SQLITE_DETERMINISTIC
#define SQLITE_DETERMINISTIC 0
#endif
#ifndef SQLITE_CALLBACK
#define SQLITE_CALLBACK
#endif
SQLITE_CALLBACK int sqlite3_snapfn_init(sqlite3* db, const char** /*pzErrMsg*/,
const sqlite3_api_routines* pApi) {
SQLITE_EXTENSION_INIT2(pApi);
int rc;
rc = sqlite3_create_function_v2(
db,
"snap", // zFunctionName
1, // nArg
SQLITE_UTF8 | SQLITE_DETERMINISTIC, // eTextRep
nullptr, // pApp
snap, // xFunc
nullptr, // xStep
nullptr, // xFinal
nullptr // xDestroy
);
if (rc != SQLITE_OK) {
return rc;
}
rc = sqlite3_create_function_v2(
db,
"unsnap", // zFunctionName
1, // nArg
SQLITE_UTF8 | SQLITE_DETERMINISTIC, // eTextRep
nullptr, // pApp
unsnap, // xFunc
nullptr, // xStep
nullptr, // xFinal
nullptr // xDestroy
);
if (rc != SQLITE_OK) {
return rc;
}
return SQLITE_OK;
}
} // extern "C"
tensorflow/core/lib/db/sqlite.cc
+301
View File
@@ -0,0 +1,301 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/lib/db/sqlite.h"
#include <cstdlib>
#include <string>
#include "absl/log/check.h"
#include "absl/log/log.h"
#include "absl/status/status.h"
#include "absl/strings/ascii.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/string_view.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/strcat.h"
#include "tensorflow/core/platform/stringpiece.h"
#include "tensorflow/core/platform/stringprintf.h"
#include "tensorflow/core/platform/types.h"
#include "tsl/platform/errors.h"
#include "tsl/platform/macros.h"
#include "tsl/platform/status.h"
extern "C" int sqlite3_snapfn_init(sqlite3*, const char**, const void*);
namespace tensorflow {
namespace {
absl::StatusCode GetTfErrorCode(int code) {
// See: https://sqlite.org/rescode.html
switch (code & 0xff) {
case SQLITE_OK: // Successful result
case SQLITE_ROW: // Step has another row ready
case SQLITE_DONE: // Step has finished executing
return absl::StatusCode::kOk;
case SQLITE_ABORT: // Callback routine requested an abort
return absl::StatusCode::kAborted;
case SQLITE_READONLY: // Attempt to write a readonly database
case SQLITE_MISMATCH: // Data type mismatch
return absl::StatusCode::kFailedPrecondition;
case SQLITE_MISUSE: // Library used incorrectly
case SQLITE_INTERNAL: // Internal logic error in SQLite
return absl::StatusCode::kInternal;
case SQLITE_RANGE: // 2nd parameter to sqlite3_bind out of range
return absl::StatusCode::kOutOfRange;
case SQLITE_CANTOPEN: // Unable to open the database file
case SQLITE_CONSTRAINT: // Abort due to constraint violation
case SQLITE_NOTFOUND: // Unknown opcode or statement parameter name
case SQLITE_NOTADB: // File opened that is not a database file
return absl::StatusCode::kInvalidArgument;
case SQLITE_CORRUPT: // The database disk image is malformed
return absl::StatusCode::kDataLoss;
case SQLITE_AUTH: // Authorization denied
case SQLITE_PERM: // Access permission denied
return absl::StatusCode::kPermissionDenied;
case SQLITE_FULL: // Insertion failed because database is full
case SQLITE_TOOBIG: // String or BLOB exceeds size limit
case SQLITE_NOLFS: // Uses OS features not supported on host
return absl::StatusCode::kResourceExhausted;
case SQLITE_BUSY: // The database file is locked
case SQLITE_LOCKED: // A table in the database is locked
case SQLITE_PROTOCOL: // Database lock protocol error
case SQLITE_NOMEM: // Out of heap or perhaps lookaside memory
return absl::StatusCode::kUnavailable;
case SQLITE_INTERRUPT: // Operation terminated by sqlite3_interrupt
return absl::StatusCode::kCancelled;
case SQLITE_ERROR: // SQL error or missing database
case SQLITE_IOERR: // Some kind of disk I/O error occurred
case SQLITE_SCHEMA: // The database schema changed
default:
return absl::StatusCode::kUnknown;
}
}
template <typename... Args>
absl::Status PrintfStatus(int rc, const absl::FormatSpec<Args...>& fmt,
Args&&... args) {
return {GetTfErrorCode(rc),
strings::Printf(fmt, std::forward<Args>(args)...)};
}
sqlite3_stmt* PrepareRawOrDie(sqlite3* db, const char* sql) {
sqlite3_stmt* stmt = nullptr;
int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr);
CHECK_EQ(SQLITE_OK, rc) << sql;
return stmt;
}
absl::Status SetPragma(Sqlite* db, const char* pragma,
const absl::string_view& value) {
if (value.empty()) return absl::OkStatus();
for (auto p = value.begin(); p < value.end(); ++p) {
if (!(absl::ascii_isalnum(*p) || *p == '-')) {
return absl::InvalidArgumentError("Illegal pragma character");
}
}
SqliteStatement stmt;
TF_RETURN_IF_ERROR( // We can't use Bind*() pragma statements.
db->Prepare(absl::StrCat("PRAGMA ", pragma, "=", value), &stmt));
bool unused_done;
return stmt.Step(&unused_done);
}
const absl::string_view GetEnv(const char* var) {
const char* val = std::getenv(var);
return (val == nullptr) ? absl::string_view() : absl::string_view(val);
}
absl::Status EnvPragma(Sqlite* db, const char* pragma, const char* var) {
TF_RETURN_WITH_CONTEXT_IF_ERROR(SetPragma(db, pragma, GetEnv(var)), "getenv(",
var, ")");
return absl::OkStatus();
}
} // namespace
/* static */
absl::Status Sqlite::Open(const std::string& path, int flags, Sqlite** db) {
flags |= SQLITE_OPEN_PRIVATECACHE;
flags |= SQLITE_OPEN_URI;
sqlite3* sqlite = nullptr;
int rc = sqlite3_open_v2(path.c_str(), &sqlite, flags, nullptr);
if (rc != SQLITE_OK) {
*db = nullptr;
return PrintfStatus(rc, "Sqlite::Open(%s) failed: %s", path.c_str(),
sqlite3_errstr(rc));
}
CHECK_EQ(SQLITE_OK, sqlite3_extended_result_codes(sqlite, 1));
CHECK_EQ(SQLITE_OK, sqlite3_snapfn_init(sqlite, nullptr, nullptr));
// Prepare these tiny privileged statements for SqliteTransaction
// so it can do less work, particularly in its constructor, per
// Google C++ Style.
sqlite3_stmt* begin = PrepareRawOrDie(sqlite, "BEGIN");
sqlite3_stmt* commit = PrepareRawOrDie(sqlite, "COMMIT");
sqlite3_stmt* rollback = PrepareRawOrDie(sqlite, "ROLLBACK");
*db = new Sqlite(sqlite, begin, commit, rollback);
absl::Status s = absl::OkStatus();
// Up until 2016 the default SQLite page_size was 1024. This ensures
// the new default regardless of linkage unless configured otherwise.
s.Update(SetPragma(*db, "page_size", "4096"));
// TensorFlow is designed to work well in all SQLite modes. However
// users might find tuning some these pragmas rewarding, depending on
// various considerations. Pragmas are set on a best-effort basis and
// might be ignored.
s.Update(EnvPragma(*db, "secure_delete", "TF_SQLITE_SECURE_DELETE"));
s.Update(EnvPragma(*db, "page_size", "TF_SQLITE_PAGE_SIZE"));
s.Update(EnvPragma(*db, "journal_mode", "TF_SQLITE_JOURNAL_MODE"));
s.Update(EnvPragma(*db, "synchronous", "TF_SQLITE_SYNCHRONOUS"));
s.Update(EnvPragma(*db, "mmap_size", "TF_SQLITE_MMAP_SIZE"));
s.Update(EnvPragma(*db, "locking_mode", "TF_SQLITE_LOCKING_MODE"));
s.Update(EnvPragma(*db, "cache_size", "TF_SQLITE_CACHE_SIZE"));
s.Update(EnvPragma(*db, "auto_vacuum", "TF_SQLITE_AUTO_VACUUM"));
DCHECK((*db)->RefCountIsOne());
if (!s.ok()) {
(*db)->Unref();
*db = nullptr;
}
return s;
}
Sqlite::~Sqlite() {
sqlite3_finalize(rollback_);
sqlite3_finalize(commit_);
sqlite3_finalize(begin_);
CHECK_EQ(SQLITE_OK, sqlite3_close(db_));
}
absl::Status Sqlite::Prepare(const absl::string_view& sql,
SqliteStatement* stmt) {
SqliteLock lock(*this);
sqlite3_stmt* ps = nullptr;
int rc = sqlite3_prepare_v2(db_, sql.data(), static_cast<int>(sql.size()),
&ps, nullptr);
if (rc != SQLITE_OK) {
*stmt = SqliteStatement();
return PrintfStatus(rc, "Prepare() failed: [%d] %s: %.*s", rc, errmsg(),
sql.size(), sql.data());
}
*stmt = SqliteStatement(this, ps);
return absl::OkStatus();
}
absl::Status SqliteStatement::Step(bool* is_done) {
DCHECK(stmt_ != nullptr);
if (TF_PREDICT_FALSE(bind_error_ != SQLITE_OK)) {
*is_done = true;
return PrintfStatus(bind_error_, "Bind(%d) failed: %s: %s",
bind_error_parameter_, sqlite3_errstr(bind_error_),
sql());
}
SqliteLock lock(*db_);
int rc = sqlite3_step(stmt_);
switch (rc) {
case SQLITE_ROW:
*is_done = false;
return absl::OkStatus();
case SQLITE_DONE:
*is_done = true;
return absl::OkStatus();
default:
*is_done = true;
return PrintfStatus(rc, "Step() failed: [%d] %s: %s", rc, db_->errmsg(),
sql());
}
}
bool SqliteStatement::StepOrDie() {
bool is_done;
TF_CHECK_OK(Step(&is_done));
return !is_done;
}
absl::Status SqliteStatement::StepOnce() {
bool is_done;
TF_RETURN_IF_ERROR(Step(&is_done));
if (TF_PREDICT_FALSE(is_done)) {
return absl::InternalError(absl::StrCat("No rows returned: ", sql()));
}
return absl::OkStatus();
}
const SqliteStatement& SqliteStatement::StepOnceOrDie() {
TF_CHECK_OK(StepOnce());
return *this;
}
absl::Status SqliteStatement::StepAndReset() {
bool is_done;
absl::Status s = Step(&is_done);
if (TF_PREDICT_FALSE(s.ok() && !is_done)) {
s = absl::InternalError(absl::StrCat("Unexpected row: ", sql()));
}
Reset();
return s;
}
void SqliteStatement::StepAndResetOrDie() { TF_CHECK_OK(StepAndReset()); }
void SqliteStatement::Reset() {
if (TF_PREDICT_TRUE(stmt_ != nullptr)) {
sqlite3_reset(stmt_);
sqlite3_clear_bindings(stmt_);
}
bind_error_ = SQLITE_OK;
size_ = 0;
}
SqliteTransaction::SqliteTransaction(Sqlite& db) : db_(&db) {
sqlite3_mutex_enter(sqlite3_db_mutex(db_->db_));
CHECK(!db_->is_in_transaction_);
db_->is_in_transaction_ = true;
Begin();
}
SqliteTransaction::~SqliteTransaction() {
// Rollback should only return an error if there's no transaction.
// Since the API performs auto-rollbacks in some cases, we ignore.
sqlite3_step(db_->rollback_);
sqlite3_reset(db_->rollback_);
sqlite3_reset(db_->begin_);
db_->is_in_transaction_ = false;
sqlite3_mutex_leave(sqlite3_db_mutex(db_->db_));
}
void SqliteTransaction::Begin() {
// This shouldn't allocate memory or perform I/O. All it does is
// execute OP_AutoCommit(0, 0) a.k.a. BEGIN DEFERRED which flips
// the sqlite3::autoCommit bit.
if (sqlite3_step(db_->begin_) != SQLITE_DONE) {
// It shouldn't be possible for this to fail since we already
// performed the reentrancy check.
LOG(FATAL) << "BEGIN failed: " << sqlite3_errmsg(db_->db_);
}
}
absl::Status SqliteTransaction::Commit() {
int rc = sqlite3_step(db_->commit_);
if (rc != SQLITE_DONE) {
return PrintfStatus(rc, "COMMIT failed: [%d] %s", rc,
sqlite3_errmsg(db_->db_));
}
sqlite3_reset(db_->commit_);
sqlite3_reset(db_->begin_);
Begin();
return absl::OkStatus();
}
} // namespace tensorflow
tensorflow/core/lib/db/sqlite.h
+458
View File
@@ -0,0 +1,458 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_DB_SQLITE_H_
#define TENSORFLOW_CORE_LIB_DB_SQLITE_H_
#include <cstddef>
#include <cstdint>
#include <mutex>
#include "absl/log/check.h"
#include "absl/strings/string_view.h"
#include "sqlite3.h"
#include "tensorflow/core/lib/core/refcount.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
#include "tsl/platform/status.h"
/// TensorFlow SQLite Veneer
///
/// - Memory safety
/// - Less boilerplate
/// - Removes deprecated stuff
/// - Pretends UTF16 doesn't exist
/// - Transaction compile-time safety
/// - Statically loads our native extensions
/// - Error reporting via tensorflow::Status et al.
///
/// SQLite>=3.8.2 needs to be supported until April 2019, which is when
/// Ubuntu 14.04 LTS becomes EOL.
namespace tensorflow {
class SqliteLock;
class SqliteStatement;
class SqliteTransaction;
/// \brief SQLite connection object.
///
/// The SQLite connection is closed automatically by the destructor.
/// Reference counting ensures that happens after its statements are
/// destructed.
///
/// Instances are reference counted and can be shared between threads.
/// This class offers the same thread safety behaviors as the SQLite
/// API itself.
///
/// This veneer uses auto-commit mode by default, which means a 4ms
/// fsync() happens after every write unless a SqliteTransaction is
/// used or WAL mode is enabled beforehand.
class TF_LOCKABLE Sqlite : public core::RefCounted {
public:
/// \brief Closes SQLite connection, which can take milliseconds.
~Sqlite() override;
/// \brief Opens SQLite database file.
///
/// Most users will want to set flags to SQLITE_OPEN_READWRITE |
/// SQLITE_OPEN_CREATE. There are many other open flags; here are
/// notes on a few of them:
///
/// - SQLITE_OPEN_READONLY: Allowed if no WAL journal is active.
/// - SQLITE_OPEN_SHAREDCACHE: Will be ignored because this veneer
/// doesn't support the unlock notify API.
/// - SQLITE_OPEN_NOMUTEX: Means access to this connection MUST be
/// serialized by the caller in accordance with the same contracts
/// implemented by this API.
///
/// This function sets PRAGMA values from TF_SQLITE_* environment
/// variables. See sqlite.cc to learn more.
static absl::Status Open(const std::string& path, int flags, Sqlite** db);
/// \brief Creates SQLite statement.
///
/// This routine should never fail if sql is valid and does not
/// reference tables. When tables are referenced, system calls are
/// needed which can take microseconds. When the schema changes, this
/// routine will retry automatically and then possibly fail.
///
/// The returned statement holds a reference to this object.
absl::Status Prepare(const absl::string_view& sql, SqliteStatement* stmt);
SqliteStatement PrepareOrDie(const absl::string_view& sql);
/// \brief Returns extended result code of last error.
///
/// If the most recent API call was successful, the result is
/// undefined. The legacy result code can be obtained by saying
/// errcode() & 0xff.
int errcode() const TF_EXCLUSIVE_LOCKS_REQUIRED(this) {
return sqlite3_extended_errcode(db_);
}
/// \brief Returns pointer to current error message state.
const char* errmsg() const TF_EXCLUSIVE_LOCKS_REQUIRED(this) {
return sqlite3_errmsg(db_);
}
/// \brief Returns rowid assigned to last successful insert.
int64_t last_insert_rowid() const TF_EXCLUSIVE_LOCKS_REQUIRED(this) {
return sqlite3_last_insert_rowid(db_);
}
/// \brief Returns number of rows directly changed by last write.
int64_t changes() const TF_EXCLUSIVE_LOCKS_REQUIRED(this) {
return sqlite3_changes(db_);
}
private:
friend class SqliteLock;
friend class SqliteStatement;
friend class SqliteTransaction;
Sqlite(sqlite3* db, sqlite3_stmt* begin, sqlite3_stmt* commit,
sqlite3_stmt* rollback) noexcept
: db_(db), begin_(begin), commit_(commit), rollback_(rollback) {}
sqlite3* const db_;
sqlite3_stmt* const begin_;
sqlite3_stmt* const commit_;
sqlite3_stmt* const rollback_;
bool is_in_transaction_ = false;
Sqlite(const Sqlite&) = delete;
void operator=(const Sqlite&) = delete;
};
/// \brief SQLite prepared statement.
///
/// Instances can only be shared between threads if caller serializes
/// access from first Bind*() to *Reset().
///
/// When reusing a statement in a loop, be certain to not have jumps
/// betwixt Bind*() and *Reset().
class SqliteStatement {
public:
/// \brief Initializes an empty statement to be assigned later.
SqliteStatement() noexcept = default;
/// \brief Finalizes statement.
///
/// This can take milliseconds if it was blocking the Sqlite
/// connection object from being freed.
~SqliteStatement() {
sqlite3_finalize(stmt_);
if (db_ != nullptr) db_->Unref();
}
/// \brief Returns true if statement is initialized.
explicit operator bool() const { return stmt_ != nullptr; }
/// \brief Returns SQL text from when this query was prepared.
const char* sql() const { return sqlite3_sql(stmt_); }
/// \brief Number of bytes bound since last *Reset().
uint64_t size() { return size_; }
/// \brief Executes query for fetching arbitrary rows.
///
/// `is_done` will always be set to true unless SQLITE_ROW is
/// returned by the underlying API. If status() is already in an
/// error state, then this method is a no-op and the existing status
/// is returned.
///
/// The OrDie version returns `!is_done` which, if true, indicates a
/// row is available.
///
/// This statement should be Reset() or destructed when finished with
/// the result.
absl::Status Step(bool* is_done);
bool StepOrDie() TF_MUST_USE_RESULT;
/// \brief Executes query when only one row is desired.
///
/// If a row isn't returned, an internal error Status is returned
/// that won't be reflected in the connection error state.
///
/// This statement should be Reset() or destructed when finished with
/// the result.
absl::Status StepOnce();
const SqliteStatement& StepOnceOrDie();
/// \brief Executes query, ensures zero rows returned, then Reset().
///
/// If a row is returned, an internal error Status is returned that
/// won't be reflected in the connection error state.
absl::Status StepAndReset();
void StepAndResetOrDie();
/// \brief Resets statement so it can be executed again.
///
/// Implementation note: This method diverges from canonical API
/// behavior by calling sqlite3_clear_bindings() in addition to
/// sqlite3_reset(). That makes the veneer safer; we haven't found a
/// super compelling reason yet to call them independently.
void Reset();
/// \brief Binds signed 64-bit integer to 1-indexed query parameter.
void BindInt(int parameter, int64_t value) {
Update(sqlite3_bind_int64(stmt_, parameter, value), parameter);
size_ += sizeof(int64_t);
}
void BindInt(const char* parameter, int64_t value) {
BindInt(GetParameterIndex(parameter), value);
}
/// \brief Binds double to 1-indexed query parameter.
void BindDouble(int parameter, double value) {
Update(sqlite3_bind_double(stmt_, parameter, value), parameter);
size_ += sizeof(double);
}
void BindDouble(const char* parameter, double value) {
BindDouble(GetParameterIndex(parameter), value);
}
/// \brief Copies UTF-8 text to 1-indexed query parameter.
///
/// If NUL characters are present, they will still go in the DB and
/// be successfully retrieved by ColumnString(); however, the
/// behavior of these values with SQLite functions is undefined.
///
/// When using the unsafe methods, the data must not be changed or
/// freed until this statement is Reset() or finalized.
void BindText(int parameter, const absl::string_view& text) {
Update(sqlite3_bind_text64(stmt_, parameter, text.data(), text.size(),
SQLITE_TRANSIENT, SQLITE_UTF8),
parameter);
size_ += text.size();
}
void BindText(const char* parameter, const absl::string_view& text) {
BindText(GetParameterIndex(parameter), text);
}
void BindTextUnsafe(int parameter, const absl::string_view& text) {
Update(sqlite3_bind_text64(stmt_, parameter, text.data(), text.size(),
SQLITE_STATIC, SQLITE_UTF8),
parameter);
size_ += text.size();
}
void BindTextUnsafe(const char* parameter, const absl::string_view& text) {
BindTextUnsafe(GetParameterIndex(parameter), text);
}
/// \brief Copies binary data to 1-indexed query parameter.
///
/// When using the unsafe methods, the data must not be changed or
/// freed until this statement is Reset() or finalized.
void BindBlob(int parameter, const absl::string_view& blob) {
Update(sqlite3_bind_blob64(stmt_, parameter, blob.data(), blob.size(),
SQLITE_TRANSIENT),
parameter);
size_ += blob.size();
}
void BindBlob(const char* parameter, const absl::string_view& blob) {
BindBlob(GetParameterIndex(parameter), blob);
}
void BindBlobUnsafe(int parameter, const absl::string_view& blob) {
Update(sqlite3_bind_blob64(stmt_, parameter, blob.data(), blob.size(),
SQLITE_STATIC),
parameter);
size_ += blob.size();
}
void BindBlobUnsafe(const char* parameter, const absl::string_view& text) {
BindBlobUnsafe(GetParameterIndex(parameter), text);
}
/// \brief Returns number of columns in result set.
int ColumnCount() const TF_MUST_USE_RESULT {
return sqlite3_column_count(stmt_);
}
/// \brief Returns type of 0-indexed column value in row data.
///
/// Please note that SQLite is dynamically typed and the type of a
/// particular column can vary from row to row.
int ColumnType(int column) const TF_MUST_USE_RESULT {
return sqlite3_column_type(stmt_, column);
}
/// \brief Returns 0-indexed column from row result coerced as an integer.
int64_t ColumnInt(int column) const TF_MUST_USE_RESULT {
return sqlite3_column_int64(stmt_, column);
}
/// \brief Returns 0-indexed column from row result coerced as a double.
double ColumnDouble(int column) const TF_MUST_USE_RESULT {
return sqlite3_column_double(stmt_, column);
}
/// \brief Copies 0-indexed column from row result coerced as a string.
///
/// NULL values are returned as empty string. This method should be
/// used for both BLOB and TEXT columns. See also: ColumnType().
std::string ColumnString(int column) const TF_MUST_USE_RESULT {
auto data = sqlite3_column_blob(stmt_, column);
if (data == nullptr) return "";
return {static_cast<const char*>(data),
static_cast<size_t>(ColumnSize(column))};
}
/// \brief Returns pointer to binary data at 0-indexed column.
///
/// Empty values are returned as NULL. The returned memory will no
/// longer be valid the next time Step() or Reset() is called. No NUL
/// terminator is added.
absl::string_view ColumnStringUnsafe(int column) const TF_MUST_USE_RESULT {
return {static_cast<const char*>(sqlite3_column_blob(stmt_, column)),
static_cast<size_t>(ColumnSize(column))};
}
/// \brief Returns number of bytes stored at 0-indexed column.
int ColumnSize(int column) const TF_MUST_USE_RESULT {
return sqlite3_column_bytes(stmt_, column);
}
/// \brief Move constructor, after which <other> is reset to empty.
SqliteStatement(SqliteStatement&& other) noexcept
: db_(other.db_), stmt_(other.stmt_), bind_error_(other.bind_error_) {
other.db_ = nullptr;
other.stmt_ = nullptr;
other.bind_error_ = SQLITE_OK;
}
/// \brief Move assignment, after which <other> is reset to empty.
SqliteStatement& operator=(SqliteStatement&& other) noexcept {
if (&other != this) {
if (db_ != nullptr) db_->Unref();
if (stmt_ != nullptr) sqlite3_finalize(stmt_);
db_ = other.db_;
stmt_ = other.stmt_;
bind_error_ = other.bind_error_;
size_ = other.size_;
other.db_ = nullptr;
other.stmt_ = nullptr;
other.bind_error_ = SQLITE_OK;
other.size_ = 0;
}
return *this;
}
private:
friend class Sqlite;
SqliteStatement(Sqlite* db, sqlite3_stmt* stmt) noexcept
: db_(db), stmt_(stmt) {
db_->Ref();
}
void Update(int rc, int parameter) {
// Binding strings can fail if they exceed length limit.
if (TF_PREDICT_FALSE(rc != SQLITE_OK)) {
if (bind_error_ == SQLITE_OK) {
bind_error_ = rc;
bind_error_parameter_ = parameter;
}
}
}
int GetParameterIndex(const char* parameter) {
int index = sqlite3_bind_parameter_index(stmt_, parameter);
DCHECK(index > 0); // OK to compile away since it'll fail again
return index;
}
Sqlite* db_ = nullptr;
sqlite3_stmt* stmt_ = nullptr;
int bind_error_ = SQLITE_OK;
int bind_error_parameter_ = 0;
uint64_t size_ = 0;
SqliteStatement(const SqliteStatement&) = delete;
void operator=(const SqliteStatement&) = delete;
};
/// \brief Reentrant SQLite connection object lock
///
/// This is a no-op if SQLITE_OPEN_NOMUTEX was used.
class TF_SCOPED_LOCKABLE SqliteLock {
public:
explicit SqliteLock(Sqlite& db) TF_EXCLUSIVE_LOCK_FUNCTION(db)
: mutex_(sqlite3_db_mutex(db.db_)) {
sqlite3_mutex_enter(mutex_);
}
SqliteLock(Sqlite& db, std::try_to_lock_t) TF_EXCLUSIVE_LOCK_FUNCTION(db)
: mutex_(sqlite3_db_mutex(db.db_)) {
if (TF_PREDICT_FALSE(sqlite3_mutex_try(mutex_) != SQLITE_OK)) {
is_locked_ = false;
}
}
~SqliteLock() TF_UNLOCK_FUNCTION() {
if (is_locked_) sqlite3_mutex_leave(mutex_);
}
explicit operator bool() const { return is_locked_; }
private:
sqlite3_mutex* const mutex_;
bool is_locked_ = true;
SqliteLock(const SqliteLock&) = delete;
void operator=(const SqliteLock&) = delete;
};
#define SqliteLock(x) static_assert(0, "sqlite_lock_decl_missing_name");
/// \brief SQLite transaction scope.
///
/// This class acquires an exclusive lock on the connection object (if
/// mutexes weren't disabled) and runs BEGIN / ROLLBACK automatically.
/// Unlike SqliteLock this scope is non-reentrant. To avoid program
/// crashes, business logic should use the TF_EXCLUSIVE_LOCK_FUNCTION and
/// TF_LOCKS_EXCLUDED annotations as much as possible.
class TF_SCOPED_LOCKABLE SqliteTransaction {
public:
/// \brief Locks db and begins deferred transaction.
///
/// This will crash if a transaction is already active.
explicit SqliteTransaction(Sqlite& db) TF_EXCLUSIVE_LOCK_FUNCTION(db);
/// \brief Runs ROLLBACK and unlocks.
~SqliteTransaction() TF_UNLOCK_FUNCTION();
/// \brief Commits transaction.
///
/// If this is successful, a new transaction will be started, which
/// is rolled back when exiting the scope.
absl::Status Commit();
private:
void Begin();
Sqlite* const db_;
SqliteTransaction(const SqliteTransaction&) = delete;
void operator=(const SqliteTransaction&) = delete;
};
#define SQLITE_EXCLUSIVE_TRANSACTIONS_REQUIRED(...) \
TF_EXCLUSIVE_LOCKS_REQUIRED(__VA_ARGS__)
#define SQLITE_TRANSACTIONS_EXCLUDED(...) TF_LOCKS_EXCLUDED(__VA_ARGS__)
inline SqliteStatement Sqlite::PrepareOrDie(const absl::string_view& sql) {
SqliteStatement stmt;
TF_CHECK_OK(Prepare(sql, &stmt));
return stmt;
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_DB_SQLITE_H_
tensorflow/core/lib/db/sqlite_test.cc
+303
View File
@@ -0,0 +1,303 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/lib/db/sqlite.h"
#include <array>
#include <climits>
#include "xla/tsl/lib/core/status_test_util.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace {
class SqliteTest : public ::testing::Test {
protected:
void SetUp() override {
TF_ASSERT_OK(Sqlite::Open(":memory:", SQLITE_OPEN_READWRITE, &db_));
db_->PrepareOrDie("CREATE TABLE T (a BLOB, b BLOB)").StepAndResetOrDie();
}
void TearDown() override { db_->Unref(); }
Sqlite* db_;
bool is_done_;
};
TEST_F(SqliteTest, InsertAndSelectInt) {
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindInt(1, 3);
stmt.BindInt(2, -7);
TF_ASSERT_OK(stmt.StepAndReset());
stmt.BindInt(1, 123);
stmt.BindInt(2, -123);
TF_ASSERT_OK(stmt.StepAndReset());
stmt = db_->PrepareOrDie("SELECT a, b FROM T ORDER BY b");
TF_ASSERT_OK(stmt.Step(&is_done_));
ASSERT_FALSE(is_done_);
EXPECT_EQ(123, stmt.ColumnInt(0));
EXPECT_EQ(-123, stmt.ColumnInt(1));
TF_ASSERT_OK(stmt.Step(&is_done_));
ASSERT_FALSE(is_done_);
EXPECT_EQ(3, stmt.ColumnInt(0));
EXPECT_EQ(-7, stmt.ColumnInt(1));
TF_ASSERT_OK(stmt.Step(&is_done_));
ASSERT_TRUE(is_done_);
}
TEST_F(SqliteTest, InsertAndSelectDouble) {
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindDouble(1, 6.28318530);
stmt.BindDouble(2, 1.61803399);
TF_ASSERT_OK(stmt.StepAndReset());
stmt = db_->PrepareOrDie("SELECT a, b FROM T");
TF_ASSERT_OK(stmt.Step(&is_done_));
EXPECT_EQ(6.28318530, stmt.ColumnDouble(0));
EXPECT_EQ(1.61803399, stmt.ColumnDouble(1));
EXPECT_EQ(6, stmt.ColumnInt(0));
EXPECT_EQ(1, stmt.ColumnInt(1));
}
#ifdef DSQLITE_ENABLE_JSON1
TEST_F(SqliteTest, Json1Extension) {
string s1 = "{\"key\": 42}";
string s2 = "{\"key\": \"value\"}";
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindText(1, s1);
stmt.BindText(2, s2);
TF_ASSERT_OK(stmt.StepAndReset());
stmt = db_->PrepareOrDie("SELECT json_extract(a, '$.key'), json_extract(b, '$.key') FROM T");
TF_ASSERT_OK(stmt.Step(&is_done_));
EXPECT_EQ(42, stmt.ColumnInt(0));
EXPECT_EQ("value", stmt.ColumnString(1));
}
#endif //DSQLITE_ENABLE_JSON1
TEST_F(SqliteTest, NulCharsInString) {
std::string s; // XXX: Want to write {2, '\0'} but not sure why not.
s.append(static_cast<size_t>(2), '\0');
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindBlob(1, s);
stmt.BindText(2, s);
TF_ASSERT_OK(stmt.StepAndReset());
stmt = db_->PrepareOrDie("SELECT a, b FROM T");
TF_ASSERT_OK(stmt.Step(&is_done_));
EXPECT_EQ(2, stmt.ColumnSize(0));
EXPECT_EQ(2, stmt.ColumnString(0).size());
EXPECT_EQ('\0', stmt.ColumnString(0).at(0));
EXPECT_EQ('\0', stmt.ColumnString(0).at(1));
EXPECT_EQ(2, stmt.ColumnSize(1));
EXPECT_EQ(2, stmt.ColumnString(1).size());
EXPECT_EQ('\0', stmt.ColumnString(1).at(0));
EXPECT_EQ('\0', stmt.ColumnString(1).at(1));
}
TEST_F(SqliteTest, Unicode) {
std::string s = "要依法治国是赞美那些谁是公义的和惩罚恶人。 - 韩非";
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindBlob(1, s);
stmt.BindText(2, s);
TF_ASSERT_OK(stmt.StepAndReset());
stmt = db_->PrepareOrDie("SELECT a, b FROM T");
TF_ASSERT_OK(stmt.Step(&is_done_));
EXPECT_EQ(s, stmt.ColumnString(0));
EXPECT_EQ(s, stmt.ColumnString(1));
}
TEST_F(SqliteTest, StepAndResetClearsBindings) {
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindInt(1, 1);
stmt.BindInt(2, 123);
TF_ASSERT_OK(stmt.StepAndReset());
stmt.BindInt(1, 2);
TF_ASSERT_OK(stmt.StepAndReset());
stmt = db_->PrepareOrDie("SELECT b FROM T ORDER BY a");
TF_ASSERT_OK(stmt.Step(&is_done_));
EXPECT_EQ(123, stmt.ColumnInt(0));
TF_ASSERT_OK(stmt.Step(&is_done_));
EXPECT_EQ(SQLITE_NULL, stmt.ColumnType(0));
}
TEST_F(SqliteTest, SafeBind) {
std::string s = "hello";
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindBlob(1, s);
stmt.BindText(2, s);
s.at(0) = 'y';
TF_ASSERT_OK(stmt.StepAndReset());
stmt = db_->PrepareOrDie("SELECT a, b FROM T");
TF_ASSERT_OK(stmt.Step(&is_done_));
EXPECT_EQ("hello", stmt.ColumnString(0));
EXPECT_EQ("hello", stmt.ColumnString(1));
}
TEST_F(SqliteTest, UnsafeBind) {
std::string s = "hello";
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindBlobUnsafe(1, s);
stmt.BindTextUnsafe(2, s);
s.at(0) = 'y';
TF_ASSERT_OK(stmt.StepAndReset());
stmt = db_->PrepareOrDie("SELECT a, b FROM T");
TF_ASSERT_OK(stmt.Step(&is_done_));
EXPECT_EQ("yello", stmt.ColumnString(0));
EXPECT_EQ("yello", stmt.ColumnString(1));
}
TEST_F(SqliteTest, UnsafeColumn) {
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindInt(1, 1);
stmt.BindText(2, "hello");
TF_ASSERT_OK(stmt.StepAndReset());
stmt.BindInt(1, 2);
stmt.BindText(2, "there");
TF_ASSERT_OK(stmt.StepAndReset());
stmt = db_->PrepareOrDie("SELECT b FROM T ORDER BY a");
TF_ASSERT_OK(stmt.Step(&is_done_));
absl::string_view p = stmt.ColumnStringUnsafe(0);
EXPECT_EQ('h', *p.data());
TF_ASSERT_OK(stmt.Step(&is_done_));
// This will actually happen, but it's not safe to test this behavior.
// EXPECT_EQ('t', *p.data());
}
TEST_F(SqliteTest, NamedParameterBind) {
auto stmt = db_->PrepareOrDie("INSERT INTO T (a) VALUES (:a)");
stmt.BindText(":a", "lol");
TF_ASSERT_OK(stmt.StepAndReset());
stmt = db_->PrepareOrDie("SELECT COUNT(*) FROM T");
TF_ASSERT_OK(stmt.Step(&is_done_));
EXPECT_EQ(1, stmt.ColumnInt(0));
stmt = db_->PrepareOrDie("SELECT a FROM T");
TF_ASSERT_OK(stmt.Step(&is_done_));
EXPECT_FALSE(is_done_);
EXPECT_EQ("lol", stmt.ColumnString(0));
}
TEST_F(SqliteTest, Statement_DefaultConstructor) {
SqliteStatement stmt;
EXPECT_FALSE(stmt);
stmt = db_->PrepareOrDie("INSERT INTO T (a) VALUES (1)");
EXPECT_TRUE(stmt);
EXPECT_TRUE(stmt.StepAndReset().ok());
}
TEST_F(SqliteTest, Statement_MoveConstructor) {
SqliteStatement stmt{db_->PrepareOrDie("INSERT INTO T (a) VALUES (1)")};
EXPECT_TRUE(stmt.StepAndReset().ok());
}
TEST_F(SqliteTest, Statement_MoveAssignment) {
SqliteStatement stmt1 = db_->PrepareOrDie("INSERT INTO T (a) VALUES (1)");
SqliteStatement stmt2;
EXPECT_TRUE(stmt1.StepAndReset().ok());
EXPECT_FALSE(stmt2);
stmt2 = std::move(stmt1);
EXPECT_TRUE(stmt2.StepAndReset().ok());
}
TEST_F(SqliteTest, PrepareFailed) {
SqliteLock lock(*db_);
SqliteStatement stmt;
absl::Status s = db_->Prepare("SELECT", &stmt);
ASSERT_FALSE(s.ok());
EXPECT_NE(std::string::npos, s.message().find("SELECT"));
EXPECT_EQ(SQLITE_ERROR, db_->errcode());
}
TEST_F(SqliteTest, BindFailed) {
auto stmt = db_->PrepareOrDie("INSERT INTO T (a) VALUES (123)");
stmt.BindInt(1, 123);
absl::Status s = stmt.StepOnce();
EXPECT_NE(std::string::npos,
s.message().find("INSERT INTO T (a) VALUES (123)"))
<< s.message();
}
TEST_F(SqliteTest, SnappyExtension) {
auto stmt = db_->PrepareOrDie("SELECT UNSNAP(SNAP(?))");
stmt.BindText(1, "hello");
EXPECT_EQ("hello", stmt.StepOnceOrDie().ColumnString(0));
}
TEST_F(SqliteTest, SnappyBinaryCompatibility) {
EXPECT_EQ(
"today is the end of the republic",
db_->PrepareOrDie("SELECT UNSNAP(X'03207C746F6461792069732074686520656E64"
"206F66207468652072657075626C6963')")
.StepOnceOrDie()
.ColumnString(0));
}
TEST(SqliteOpenTest, CloseConnectionBeforeStatement_KeepsConnectionOpen) {
Sqlite* db;
TF_ASSERT_OK(Sqlite::Open(":memory:", SQLITE_OPEN_READWRITE, &db));
SqliteStatement stmt = db->PrepareOrDie("SELECT ? + ?");
db->Unref();
stmt.BindInt(1, 7);
stmt.BindInt(2, 3);
EXPECT_EQ(10, stmt.StepOnceOrDie().ColumnInt(0));
}
TEST_F(SqliteTest, TransactionRollback) {
{
SqliteTransaction txn(*db_);
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindDouble(1, 6.28318530);
stmt.BindDouble(2, 1.61803399);
TF_ASSERT_OK(stmt.StepAndReset());
}
EXPECT_EQ(
0,
db_->PrepareOrDie("SELECT COUNT(*) FROM T").StepOnceOrDie().ColumnInt(0));
}
TEST_F(SqliteTest, TransactionCommit) {
{
SqliteTransaction txn(*db_);
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindDouble(1, 6.28318530);
stmt.BindDouble(2, 1.61803399);
TF_ASSERT_OK(stmt.StepAndReset());
TF_ASSERT_OK(txn.Commit());
}
EXPECT_EQ(
1,
db_->PrepareOrDie("SELECT COUNT(*) FROM T").StepOnceOrDie().ColumnInt(0));
}
TEST_F(SqliteTest, TransactionCommitMultipleTimes) {
{
SqliteTransaction txn(*db_);
auto stmt = db_->PrepareOrDie("INSERT INTO T (a, b) VALUES (?, ?)");
stmt.BindDouble(1, 6.28318530);
stmt.BindDouble(2, 1.61803399);
TF_ASSERT_OK(stmt.StepAndReset());
TF_ASSERT_OK(txn.Commit());
stmt.BindDouble(1, 6.28318530);
stmt.BindDouble(2, 1.61803399);
TF_ASSERT_OK(stmt.StepAndReset());
TF_ASSERT_OK(txn.Commit());
}
EXPECT_EQ(
2,
db_->PrepareOrDie("SELECT COUNT(*) FROM T").StepOnceOrDie().ColumnInt(0));
}
} // namespace
} // namespace tensorflow
tensorflow/core/lib/gif/BUILD
+97
View File
@@ -0,0 +1,97 @@
# Description:
# gif io package.
load(
"//tensorflow:tensorflow.bzl",
"if_android",
"if_mobile",
"tf_cc_test",
"tf_copts",
)
load(
"//tensorflow/core/platform:rules_cc.bzl",
"cc_library",
)
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = ["//tensorflow/core:__pkg__"],
licenses = ["notice"],
)
filegroup(
name = "hdrs",
srcs = ["gif_io.h"],
visibility = ["//visibility:public"],
)
cc_library(
name = "gif_internal",
srcs = [
"gif_io.cc",
],
hdrs = ["gif_io.h"],
copts = tf_copts(),
linkopts = select({
"//tensorflow:freebsd": [],
"//tensorflow:windows": [],
"//conditions:default": ["-ldl"],
}),
deps = [
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core/platform:gif",
"@com_google_absl//absl/strings",
"@gif",
],
)
cc_library(
name = "portable_gif_internal",
srcs = if_mobile([
"gif_io.cc",
"//tensorflow/core/platform:gif_hdrs",
]),
hdrs = [
"gif_io.h",
"//tensorflow/core/lib/core:legacy_lib_core_stringpiece_header",
"//tensorflow/core/lib/gtl:legacy_android_gif_internal_headers",
"//tensorflow/core/platform:gif_internal_hdrs",
"@tsl//tsl/platform:gif_internal_hdrs",
],
copts = tf_copts(),
features = ["-layering_check"],
linkopts = if_android(["-ldl"]),
deps = [
"//tensorflow/core/platform:byte_order",
"//tensorflow/core/platform:cord",
"//tensorflow/core/platform:dynamic_annotations",
"//tensorflow/core/platform:gif",
"//tensorflow/core/platform:logging",
"//tensorflow/core/platform:stringpiece",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/base:dynamic_annotations",
"@com_google_absl//absl/strings",
"@eigen_archive//:eigen3",
"@gif",
"@tsl//tsl/platform:logging",
"@tsl//tsl/platform:tstring",
"@tsl//tsl/platform:types",
"@xla//xla/tsl/platform:types",
],
)
tf_cc_test(
name = "lib_gif_io_test",
srcs = ["gif_io_test.cc"],
data = ["//tensorflow/core/lib/gif/testdata:gif_testdata"],
deps = [
":gif_internal",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core/lib/png:png_io",
"@com_google_absl//absl/base",
],
)
tensorflow/core/lib/gif/gif_io.cc
+215
View File
@@ -0,0 +1,215 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Functions to read images in GIF format.
#include "tensorflow/core/lib/gif/gif_io.h"
#include <algorithm>
#include "absl/strings/str_cat.h"
#include "gif_lib.h" // from @gif
#include "tensorflow/core/lib/gtl/cleanup.h"
#include "tensorflow/core/platform/gif.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/mem.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace gif {
struct InputBufferInfo {
const uint8_t* buf;
int bytes_left;
};
int input_callback(GifFileType* gif_file, GifByteType* buf, int size) {
InputBufferInfo* const info =
reinterpret_cast<InputBufferInfo*>(gif_file->UserData);
if (info != nullptr) {
if (size > info->bytes_left) size = info->bytes_left;
memcpy(buf, info->buf, size);
info->buf += size;
info->bytes_left -= size;
return size;
}
return 0;
}
static const char* GifErrorStringNonNull(int error_code) {
const char* error_string = GifErrorString(error_code);
if (error_string == nullptr) {
return "Unknown error";
}
return error_string;
}
uint8_t* Decode(
const void* srcdata, int datasize,
const std::function<uint8_t*(int, int, int, int)>& allocate_output,
std::string* error_string, bool expand_animations) {
int error_code = D_GIF_SUCCEEDED;
InputBufferInfo info = {reinterpret_cast<const uint8_t*>(srcdata), datasize};
/// NOTE: After this, gif file is mostly not initialized!
GifFileType* gif_file =
DGifOpen(static_cast<void*>(&info), &input_callback, &error_code);
const auto cleanup = gtl::MakeCleanup([gif_file]() {
int error_code = D_GIF_SUCCEEDED;
if (gif_file && DGifCloseFile(gif_file, &error_code) != GIF_OK) {
LOG(WARNING) << "Fail to close gif file, reason: "
<< GifErrorStringNonNull(error_code);
}
});
if (error_code != D_GIF_SUCCEEDED) {
*error_string = absl::StrCat("failed to open gif file: ",
GifErrorStringNonNull(error_code));
return nullptr;
}
if (DGifSlurp(gif_file) != GIF_OK) {
*error_string = absl::StrCat("failed to slurp gif file: ",
GifErrorStringNonNull(gif_file->Error));
// Stop load if no images are detected or the allocation of the last image
// buffer was failed.
if (gif_file->ImageCount <= 0 ||
gif_file->SavedImages[gif_file->ImageCount - 1].RasterBits == nullptr) {
return nullptr;
}
LOG(ERROR) << *error_string;
}
int target_num_frames = gif_file->ImageCount;
if (target_num_frames <= 0) {
*error_string = "gif file does not contain any image";
return nullptr;
}
// Don't request more memory than needed for each frame, preventing OOM
int max_frame_width = 0;
int max_frame_height = 0;
for (int k = 0; k < target_num_frames; k++) {
SavedImage* si = &gif_file->SavedImages[k];
if (max_frame_height < si->ImageDesc.Height)
max_frame_height = si->ImageDesc.Height;
if (max_frame_width < si->ImageDesc.Width)
max_frame_width = si->ImageDesc.Width;
}
const int width = max_frame_width;
const int height = max_frame_height;
const int channel = 3;
if (!expand_animations) target_num_frames = 1;
uint8_t* const dstdata =
allocate_output(target_num_frames, width, height, channel);
if (!dstdata) return nullptr;
for (int64_t k = 0; k < target_num_frames; k++) {
uint8_t* this_dst = dstdata + k * width * channel * height;
SavedImage* this_image = &gif_file->SavedImages[k];
GifImageDesc* img_desc = &this_image->ImageDesc;
// The Graphics Control Block tells us which index in the color map
// correspond to "transparent color", i.e. no need to update the pixel
// on the canvas. The "transparent color index" is specific to each
// sub-frame.
GraphicsControlBlock gcb;
DGifSavedExtensionToGCB(gif_file, k, &gcb);
int imgLeft = img_desc->Left;
int imgTop = img_desc->Top;
int imgRight = img_desc->Left + img_desc->Width;
int imgBottom = img_desc->Top + img_desc->Height;
if (k > 0) {
uint8_t* last_dst = dstdata + (k - 1) * width * channel * height;
for (int64_t i = 0; i < height; ++i) {
uint8_t* p_dst = this_dst + i * width * channel;
uint8_t* l_dst = last_dst + i * width * channel;
for (int64_t j = 0; j < width; ++j) {
p_dst[j * channel + 0] = l_dst[j * channel + 0];
p_dst[j * channel + 1] = l_dst[j * channel + 1];
p_dst[j * channel + 2] = l_dst[j * channel + 2];
}
}
}
if (img_desc->Left != 0 || img_desc->Top != 0 || img_desc->Width != width ||
img_desc->Height != height) {
// If the first frame does not fill the entire canvas then fill the
// unoccupied canvas with zeros (black).
if (k == 0) {
for (int64_t i = 0; i < height; ++i) {
uint8_t* p_dst = this_dst + i * width * channel;
for (int64_t j = 0; j < width; ++j) {
p_dst[j * channel + 0] = 0;
p_dst[j * channel + 1] = 0;
p_dst[j * channel + 2] = 0;
}
}
}
imgLeft = std::max(imgLeft, 0);
imgTop = std::max(imgTop, 0);
imgRight = std::min(imgRight, width);
imgBottom = std::min(imgBottom, height);
}
ColorMapObject* color_map = this_image->ImageDesc.ColorMap
? this_image->ImageDesc.ColorMap
: gif_file->SColorMap;
if (color_map == nullptr) {
*error_string = absl::StrCat("missing color map for frame ", k);
return nullptr;
}
for (int64_t i = imgTop; i < imgBottom; ++i) {
uint8_t* p_dst = this_dst + i * width * channel;
for (int64_t j = imgLeft; j < imgRight; ++j) {
GifByteType color_index =
this_image->RasterBits[(i - img_desc->Top) * (img_desc->Width) +
(j - img_desc->Left)];
if (color_index == gcb.TransparentColor) {
// Use the pixel from the previous frame, or 0 if there was no
// previous frame.
if (k == 0) {
p_dst[j * channel + 0] = 0;
p_dst[j * channel + 1] = 0;
p_dst[j * channel + 2] = 0;
}
continue;
}
if (color_index >= color_map->ColorCount) {
*error_string = absl::StrCat("found color index ", color_index,
" outside of color map range ",
color_map->ColorCount);
return nullptr;
}
const GifColorType& gif_color = color_map->Colors[color_index];
p_dst[j * channel + 0] = gif_color.Red;
p_dst[j * channel + 1] = gif_color.Green;
p_dst[j * channel + 2] = gif_color.Blue;
}
}
}
return dstdata;
}
} // namespace gif
} // namespace tensorflow
tensorflow/core/lib/gif/gif_io.h
+53
View File
@@ -0,0 +1,53 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Functions to read and write images in GIF format.
//
// The advantage over image/codec/png{enc,dec}oder.h is that this library
// supports both 8 and 16 bit images.
//
// The decoding routine accepts binary image data as a StringPiece. These are
// implicitly constructed from strings or char* so they're completely
// transparent to the caller. They're also very cheap to construct so this
// doesn't introduce any additional overhead.
//
// The primary benefit of StringPieces being, in this case, that APIs already
// returning StringPieces (e.g., Bigtable Scanner) or Cords (e.g., IOBuffer;
// only when they're flat, though) or protocol buffer fields typed to either of
// these can be decoded without copying the data into a C++ string.
#ifndef TENSORFLOW_CORE_LIB_GIF_GIF_IO_H_
#define TENSORFLOW_CORE_LIB_GIF_GIF_IO_H_
#include <functional>
#include <string>
#include <utility>
#include <vector>
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace gif {
uint8_t* Decode(
const void* srcdata, int datasize,
const std::function<uint8_t*(int, int, int, int)>& allocate_output,
std::string* error_string, bool expand_animations = true);
} // namespace gif
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GIF_GIF_IO_H_
tensorflow/core/lib/gif/gif_io_test.cc
+264
View File
@@ -0,0 +1,264 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/lib/gif/gif_io.h"
#include <memory>
#include "tensorflow/core/lib/png/png_io.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/test.h"
namespace tensorflow {
namespace gif {
namespace {
const char kTestData[] = "tensorflow/core/lib/gif/testdata/";
struct DecodeGifTestCase {
const std::string filepath;
const int num_frames;
const int width;
const int height;
const int channels;
};
void ReadFileToStringOrDie(Env* env, const std::string& filename,
std::string* output) {
TF_CHECK_OK(ReadFileToString(env, filename, output));
}
void TestDecodeGif(Env* env, DecodeGifTestCase testcase) {
std::string gif;
ReadFileToStringOrDie(env, testcase.filepath, &gif);
// Decode gif image data.
std::unique_ptr<uint8_t[]> imgdata;
int nframes, w, h, c;
std::string error_string;
imgdata.reset(gif::Decode(
gif.data(), gif.size(),
[&](int frame_cnt, int width, int height, int channels) -> uint8_t* {
nframes = frame_cnt;
w = width;
h = height;
c = channels;
return new uint8_t[static_cast<int64_t>(frame_cnt) * height * width *
channels];
},
&error_string));
ASSERT_NE(imgdata, nullptr);
// Make sure the decoded information matches the ground-truth image info.
ASSERT_EQ(nframes, testcase.num_frames);
ASSERT_EQ(w, testcase.width);
ASSERT_EQ(h, testcase.height);
ASSERT_EQ(c, testcase.channels);
}
TEST(GifTest, Gif) {
Env* env = Env::Default();
const std::string testdata_path = kTestData;
std::vector<DecodeGifTestCase> testcases(
{// file_path, num_of_channels, width, height, channels
{testdata_path + "lena.gif", 1, 51, 26, 3},
{testdata_path + "optimized.gif", 12, 20, 40, 3},
{testdata_path + "red_black.gif", 1, 16, 16, 3},
{testdata_path + "scan.gif", 12, 20, 40, 3},
{testdata_path + "squares.gif", 2, 16, 16, 3},
{testdata_path + "3g_multiframe.gif", 519, 1920, 1080, 3}});
for (const auto& tc : testcases) {
TestDecodeGif(env, tc);
}
}
void TestDecodeAnimatedGif(Env* env, const uint8_t* gif_data,
const std::string& png_filepath, int frame_idx) {
std::string png; // ground-truth
ReadFileToStringOrDie(env, png_filepath, &png);
// Compare decoded gif to ground-truth image frames in png format.
png::DecodeContext decode;
png::CommonInitDecode(png, 3, 8, &decode);
const int width = static_cast<int>(decode.width);
const int height = static_cast<int>(decode.height);
std::unique_ptr<uint8_t[]> png_imgdata(
new uint8_t[height * width * decode.channels]);
png::CommonFinishDecode(reinterpret_cast<png_bytep>(png_imgdata.get()),
decode.channels * width * sizeof(uint8_t), &decode);
int frame_len = width * height * decode.channels;
int gif_idx = frame_len * frame_idx;
for (int i = 0; i < frame_len; i++) {
ASSERT_EQ(gif_data[gif_idx + i], png_imgdata[i]);
}
}
TEST(GifTest, AnimatedGif) {
Env* env = Env::Default();
const std::string testdata_path = kTestData;
// Read animated gif file once.
std::string gif;
ReadFileToStringOrDie(env, testdata_path + "pendulum_sm.gif", &gif);
std::unique_ptr<uint8_t[]> gif_imgdata;
int nframes, w, h, c;
std::string error_string;
gif_imgdata.reset(gif::Decode(
gif.data(), gif.size(),
[&](int num_frames, int width, int height, int channels) -> uint8_t* {
nframes = num_frames;
w = width;
h = height;
c = channels;
return new uint8_t[num_frames * height * width * channels];
},
&error_string));
TestDecodeAnimatedGif(env, gif_imgdata.get(),
testdata_path + "pendulum_sm_frame0.png", 0);
TestDecodeAnimatedGif(env, gif_imgdata.get(),
testdata_path + "pendulum_sm_frame1.png", 1);
TestDecodeAnimatedGif(env, gif_imgdata.get(),
testdata_path + "pendulum_sm_frame2.png", 2);
}
void TestExpandAnimations(Env* env, const std::string& filepath) {
std::string gif;
ReadFileToStringOrDie(env, filepath, &gif);
std::unique_ptr<uint8_t[]> imgdata;
std::string error_string;
int nframes;
// `expand_animations` is set to true by default. Set to false.
bool expand_animations = false;
imgdata.reset(gif::Decode(
gif.data(), gif.size(),
[&](int frame_cnt, int width, int height, int channels) -> uint8_t* {
nframes = frame_cnt;
return new uint8_t[frame_cnt * height * width * channels];
},
&error_string, expand_animations));
// Check that only 1 frame is being decoded.
ASSERT_EQ(nframes, 1);
}
TEST(GifTest, ExpandAnimations) {
Env* env = Env::Default();
const std::string testdata_path = kTestData;
// Test all animated gif test images.
TestExpandAnimations(env, testdata_path + "scan.gif");
TestExpandAnimations(env, testdata_path + "pendulum_sm.gif");
TestExpandAnimations(env, testdata_path + "squares.gif");
}
void TestInvalidGifFormat(const std::string& header_bytes) {
std::unique_ptr<uint8_t[]> imgdata;
std::string error_string;
int nframes;
imgdata.reset(gif::Decode(
header_bytes.data(), header_bytes.size(),
[&](int frame_cnt, int width, int height, int channels) -> uint8_t* {
nframes = frame_cnt;
return new uint8_t[frame_cnt * height * width * channels];
},
&error_string));
// Check that decoding image formats other than gif throws an error.
std::string err_msg = "failed to open gif file";
ASSERT_EQ(error_string.substr(0, 23), err_msg);
}
TEST(GifTest, BadGif) {
// Input header bytes of other image formats to gif decoder.
TestInvalidGifFormat("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"); // png
TestInvalidGifFormat("\x42\x4d"); // bmp
TestInvalidGifFormat("\xff\xd8\xff"); // jpeg
TestInvalidGifFormat("\x49\x49\x2A\x00"); // tiff
}
TEST(GifTest, TransparentIndexOutsideColorTable) {
// Given a GIF with a transparent index outside of its color table...
unsigned char encoded[43] = {
'G', 'I', 'F', '8', '9', 'a', // Header.
3, 0, 1, 0, // Logical width = 3 and height = 1.
0b1'111'0'000, // Global color table present, (7 + 1) bit
// color, unsorted, 2^(0 + 1) palette size.
0, // Background index = 0
0, // Default aspect ratio.
0x80, 0x00, 0x00, // Palette entry 0: red.
0xFF, 0xFF, 0xFF, // Palette entry 1: white.
'!', 0xF9, 0x04, // Graphic Control Extension.
1, // Transparent index is specified.
0, 0, // Delay of 0 seconds.
2, // Transparent index is 2.
0, // End GCE block.
',', 0, 0, 0, 0, // Image at logical (0, 0).
3, 0, 1, 0, // Width = 3, height = 1
0, // No local color table.
2, // Symbols need 2 bits to cover [0, 2].
2, // Two bytes of image data.
0b01'000'100, // Clear (100), 0, 1 (truncated).
0b0'101'010'0, // 1 (continued), 2, End (101), padding.
0, ';' // End of data, end of file.
};
// ...decoding that image...
std::unique_ptr<uint8_t[]> imgdata;
std::string error_string;
int nframes;
auto allocate_image_data = [&](int frame_cnt, int width, int height,
int channels) -> uint8_t* {
nframes = frame_cnt;
// Create the unique_ptr here, as gif::Decode does not return a pointer to
// the allocated array in the case of an error.
imgdata =
std::make_unique<uint8_t[]>(frame_cnt * height * width * channels);
return imgdata.get();
};
gif::Decode(encoded, sizeof(encoded), allocate_image_data, &error_string);
// ...should be successful and treat the pixels with the transparent index as
// transparent.
ASSERT_EQ(nframes, 1);
ASSERT_EQ(error_string, "");
uint8_t expected[9] = {
0x80, 0x00, 0x00, // Red (palette entry 0).
0xFF, 0xFF, 0xFF, // White (palette entry 1).
0x00, 0x00, 0x00, // Transparent (not in palette, specified by Graphic
// Control Extension), defaults to black.
};
for (int i = 0; i < 9; i++) {
ASSERT_EQ(imgdata[i], expected[i]) << "i=" << i;
}
// However, if there is an out-of-palette pixel that is not the transparent
// index...
encoded[40] = 0b0'101'011'0; // The '011' is an out-of-palette color 3.
// ...decoding the image...
error_string.clear();
gif::Decode(encoded, sizeof(encoded), allocate_image_data, &error_string);
// ...should fail with an error about a color out of range.
ASSERT_EQ(error_string, "found color index 3 outside of color map range 2");
}
} // namespace
} // namespace gif
} // namespace tensorflow
tensorflow/core/lib/gif/testdata/3g_multiframe.gif
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 22 KiB

tensorflow/core/lib/gif/testdata/BUILD
Vendored
+30
View File
@@ -0,0 +1,30 @@
# Description:
# gif test data packages.
load("//tensorflow:tensorflow.default.bzl", "filegroup")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
filegroup(
name = "gif_testdata",
srcs = [
# GIF data
"lena.gif",
"scan.gif",
"red_black.gif",
"squares.gif",
"3g_multiframe.gif",
"pendulum_sm.gif",
# Add groundtruth frames for `pendulum_sm.gif`.
# PNG format because it's lossless.
"pendulum_sm_frame0.png",
"pendulum_sm_frame1.png",
"pendulum_sm_frame2.png",
# GIF data with optimization
"optimized.gif",
],
visibility = ["//visibility:public"],
)
tensorflow/core/lib/gif/testdata/lena.gif
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.3 KiB

tensorflow/core/lib/gif/testdata/optimized.gif
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.0 KiB

tensorflow/core/lib/gif/testdata/pendulum_sm.gif
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 17 KiB

tensorflow/core/lib/gif/testdata/pendulum_sm_frame0.png
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.6 KiB

tensorflow/core/lib/gif/testdata/pendulum_sm_frame1.png
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.7 KiB

tensorflow/core/lib/gif/testdata/pendulum_sm_frame2.png
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.7 KiB

tensorflow/core/lib/gif/testdata/red_black.gif
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 88 B

tensorflow/core/lib/gif/testdata/scan.gif
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.3 KiB

tensorflow/core/lib/gif/testdata/squares.gif
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 103 B

tensorflow/core/lib/gtl/BUILD
+278
View File
@@ -0,0 +1,278 @@
load("//tensorflow:tensorflow.default.bzl", "filegroup")
load(
"//tensorflow/core/platform:rules_cc.bzl",
"cc_library",
)
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = [
# tensorflow/core:lib effectively exposes all targets under tensorflow/core/lib/**
"//tensorflow/core:__pkg__",
# tensorflow/core/lib/random uses on gtl:array_slice
"//tensorflow/core/lib/random:__pkg__",
# tensorflow/core/lib/histogram uses array_slice
"//tensorflow/core/lib/histogram:__pkg__",
# tensorflow/core/framework uses array_slice, map_util, and flatmap
"//tensorflow/core/framework:__pkg__",
# tensorflow/core/util uses array_slice, inlined_vector
"//tensorflow/core/util:__pkg__",
# tensorflow/core/tfrt/utils uses array_slice, inlined_vector
"//tensorflow/core/tfrt/utils:__pkg__",
# tensorflow/examples/custom_ops_doc/simple_hash_table uses map_util
"//tensorflow/examples/custom_ops_doc/simple_hash_table:__pkg__",
# tensorflow/core/profiler/convert uses map_util
"@org_xprof//xprof/convert:__pkg__",
# tensorflow/python/util uses array_slice
"//tensorflow/python/util:__pkg__",
# tensorflow/python uses array_slice
"//tensorflow/python:__pkg__",
],
licenses = ["notice"],
)
# Todo(bmzhao): Remaining targets to add to this BUILD file are: all tests.
cc_library(
name = "array_slice",
hdrs = ["array_slice.h"],
deps = [
":inlined_vector",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/types:span",
],
)
cc_library(
name = "cleanup",
hdrs = ["cleanup.h"],
visibility = ["//visibility:public"],
deps = ["//tensorflow/core/platform:macros"],
)
cc_library(
name = "compactptrset",
hdrs = ["compactptrset.h"],
deps = [
"@xla//xla/tsl/lib/gtl:compactptrset",
],
)
cc_library(
name = "edit_distance",
hdrs = ["edit_distance.h"],
deps = [
"//tensorflow/core/lib/gtl:array_slice",
"//tensorflow/core/lib/gtl:inlined_vector",
],
)
cc_library(
name = "flatmap",
hdrs = ["flatmap.h"],
visibility = [
"//tensorflow/core:__pkg__",
"//tensorflow/core/framework:__pkg__",
"//tensorflow/core/lib/histogram:__pkg__",
"//tensorflow/core/lib/random:__pkg__",
],
deps = [
"//tensorflow/core/lib/gtl:flatrep",
"//tensorflow/core/lib/hash",
"//tensorflow/core/platform:logging",
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/gtl:flatmap",
],
)
cc_library(
name = "flatrep",
hdrs = ["flatrep.h"],
deps = [
"@xla//xla/tsl/lib/gtl:flatrep",
],
)
cc_library(
name = "flatset",
hdrs = ["flatset.h"],
deps = [
"@xla//xla/tsl/lib/gtl:flatset",
],
)
cc_library(
name = "inlined_vector",
hdrs = ["inlined_vector.h"],
deps = [
"//tensorflow/core/platform:macros",
"//tensorflow/core/platform:types",
"@com_google_absl//absl/container:inlined_vector",
"@xla//xla/tsl/lib/gtl:inlined_vector",
],
)
cc_library(
name = "int_type",
hdrs = ["int_type.h"],
deps = [
"@xla//xla/tsl/lib/gtl:int_type",
],
)
cc_library(
name = "iterator_range",
hdrs = ["iterator_range.h"],
deps = [
"@xla//xla/tsl/lib/gtl:iterator_range",
],
)
cc_library(
name = "manual_constructor",
hdrs = ["manual_constructor.h"],
deps = [
"//tensorflow/core/platform:macros",
"//tensorflow/core/platform:platform_port",
],
)
cc_library(
name = "map_util",
srcs = [
"map_util.h",
"//tensorflow/core/lib/gtl/subtle:map_traits",
],
hdrs = ["map_util.h"],
deps = [
"//tensorflow/core/platform:hash", # TODO(dduneavy) examples/custom_ops_doc transitively depends on this
"@xla//xla/tsl/lib/gtl:map_util",
],
)
cc_library(
name = "priority_queue_util",
hdrs = ["priority_queue_util.h"],
)
cc_library(
name = "top_n",
hdrs = ["top_n.h"],
deps = ["//tensorflow/core/platform:logging"],
)
filegroup(
name = "legacy_lib_gtl_headers",
srcs = [
"array_slice.h",
"cleanup.h",
"compactptrset.h",
"edit_distance.h",
"flatmap.h",
"flatset.h",
"inlined_vector.h",
"iterator_range.h",
"priority_queue_util.h",
"@xla//xla/tsl/lib/gtl:legacy_lib_gtl_headers",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_internal_public_gtl_headers",
srcs = [
"edit_distance.h",
"int_type.h",
"manual_constructor.h",
"map_util.h",
"top_n.h",
"@xla//xla/tsl/lib/gtl:legacy_lib_internal_public_gtl_headers",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_test_internal_headers",
srcs = [
"manual_constructor.h",
"@xla//xla/tsl/lib/gtl:legacy_lib_test_internal_headers",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_android_gif_internal_headers",
srcs = [
"cleanup.h",
"@xla//xla/tsl/lib/gtl:legacy_android_gif_internal_headers",
],
visibility = [
"//tensorflow/core:__pkg__",
"//tensorflow/core/lib/gif:__pkg__",
],
)
# Export source files needed for mobile builds, which do not use granular targets.
filegroup(
name = "mobile_hdrs_no_runtime",
srcs = [
"array_slice.h",
"flatmap.h",
"flatrep.h",
"inlined_vector.h",
"top_n.h",
"@xla//xla/tsl/lib/gtl:mobile_hdrs_no_runtime",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "mobile_hdrs_only_runtime",
srcs = [
"cleanup.h",
"edit_distance.h",
"flatset.h",
"int_type.h",
"iterator_range.h",
"manual_constructor.h",
"map_util.h",
"priority_queue_util.h",
"//tensorflow/core/lib/gtl/subtle:map_traits",
"@xla//xla/tsl/lib/gtl:mobile_hdrs_only_runtime",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_gtl_all_headers",
srcs = [
"array_slice.h",
"cleanup.h",
"compactptrset.h",
"edit_distance.h",
"flatmap.h",
"flatrep.h",
"flatset.h",
"inlined_vector.h",
"int_type.h",
"iterator_range.h",
"manual_constructor.h",
"map_util.h",
"priority_queue_util.h",
"top_n.h",
"//tensorflow/core/lib/gtl/subtle:map_traits",
"@xla//xla/tsl/lib/gtl:legacy_lib_gtl_all_headers",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_gtl_tests",
srcs = [
"cleanup_test.cc",
"edit_distance_test.cc",
"manual_constructor_test.cc",
"top_n_test.cc",
],
visibility = ["//tensorflow/core:__pkg__"],
)
tensorflow/core/lib/gtl/array_slice.h
+37
View File
@@ -0,0 +1,37 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_GTL_ARRAY_SLICE_H_
#define TENSORFLOW_CORE_LIB_GTL_ARRAY_SLICE_H_
#include "absl/base/macros.h"
#include "absl/types/span.h"
// TODO(timshen): This is kept only because lots of targets transitively depend
// on it. Remove all targets' dependencies.
#include "tensorflow/core/lib/gtl/inlined_vector.h"
namespace tensorflow {
namespace gtl {
template <typename T>
using ArraySlice ABSL_DEPRECATE_AND_INLINE() = absl::Span<const T>;
template <typename T>
using MutableArraySlice ABSL_DEPRECATE_AND_INLINE() = absl::Span<T>;
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_ARRAY_SLICE_H_
tensorflow/core/lib/gtl/cleanup.h
+113
View File
@@ -0,0 +1,113 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// MakeCleanup(f) returns an RAII cleanup object that calls 'f' in its
// destructor. The easiest way to use MakeCleanup is with a lambda argument,
// capturing the return value in an 'auto' local variable. Most users will not
// need more sophisticated syntax than that.
//
// Example:
// void func() {
// FILE* fp = fopen("data.txt", "r");
// if (fp == nullptr) return;
// auto fp_cleaner = gtl::MakeCleanup([fp] { fclose(fp); });
// // No matter what, fclose(fp) will happen.
// DataObject d;
// while (ReadDataObject(fp, &d)) {
// if (d.IsBad()) {
// LOG(ERROR) << "Bad Data";
// return;
// }
// PushGoodData(d);
// }
// }
//
// You can use Cleanup<F> directly, instead of using MakeCleanup and auto,
// but there's rarely a reason to do that.
//
// You can call 'release()' on a Cleanup object to cancel the cleanup.
#ifndef TENSORFLOW_CORE_LIB_GTL_CLEANUP_H_
#define TENSORFLOW_CORE_LIB_GTL_CLEANUP_H_
#include <type_traits>
#include <utility>
#include "tensorflow/core/platform/macros.h"
namespace tensorflow {
namespace gtl {
// A move-only RAII object that calls a stored cleanup functor when
// destroyed. Cleanup<F> is the return type of gtl::MakeCleanup(F).
template <typename F>
class Cleanup {
public:
Cleanup() : released_(true), f_() {}
template <typename G>
explicit Cleanup(G&& f) // NOLINT
: f_(std::forward<G>(f)) {} // NOLINT(build/c++11)
Cleanup(Cleanup&& src) // NOLINT
: released_(src.is_released()), f_(src.release()) {}
// Implicitly move-constructible from any compatible Cleanup<G>.
// The source will be released as if src.release() were called.
// A moved-from Cleanup can be safely destroyed or reassigned.
template <typename G>
Cleanup(Cleanup<G>&& src) // NOLINT
: released_(src.is_released()), f_(src.release()) {}
// Assignment to a Cleanup object behaves like destroying it
// and making a new one in its place, analogous to unique_ptr
// semantics.
Cleanup& operator=(Cleanup&& src) { // NOLINT
if (!released_) f_();
released_ = src.released_;
f_ = src.release();
return *this;
}
~Cleanup() {
if (!released_) f_();
}
// Releases the cleanup function instead of running it.
// Hint: use c.release()() to run early.
F release() {
released_ = true;
return std::move(f_);
}
bool is_released() const { return released_; }
private:
static_assert(!std::is_reference<F>::value, "F must not be a reference");
bool released_ = false;
F f_;
};
template <int&... ExplicitParameterBarrier, typename F,
typename DecayF = typename std::decay<F>::type>
TF_MUST_USE_RESULT Cleanup<DecayF> MakeCleanup(F&& f) {
return Cleanup<DecayF>(std::forward<F>(f));
}
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_CLEANUP_H_
tensorflow/core/lib/gtl/cleanup_test.cc
+316
View File
@@ -0,0 +1,316 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/lib/gtl/cleanup.h"
#include <functional>
#include <type_traits>
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/platform/test_benchmark.h"
namespace tensorflow {
namespace {
using AnyCleanup = gtl::Cleanup<std::function<void()>>;
template <typename T1, typename T2>
void AssertTypeEq() {
static_assert(std::is_same<T1, T2>::value, "unexpected type");
}
TEST(CleanupTest, BasicLambda) {
std::string s = "active";
{
auto s_cleaner = gtl::MakeCleanup([&s] { s.assign("cleaned"); });
EXPECT_EQ("active", s);
}
EXPECT_EQ("cleaned", s);
}
TEST(FinallyTest, NoCaptureLambda) {
// Noncapturing lambdas are just structs and use aggregate initializers.
// Make sure MakeCleanup is compatible with that kind of initialization.
static std::string& s = *new std::string;
s.assign("active");
{
auto s_cleaner = gtl::MakeCleanup([] { s.append(" clean"); });
EXPECT_EQ("active", s);
}
EXPECT_EQ("active clean", s);
}
TEST(CleanupTest, Release) {
std::string s = "active";
{
auto s_cleaner = gtl::MakeCleanup([&s] { s.assign("cleaned"); });
EXPECT_EQ("active", s);
s_cleaner.release();
}
EXPECT_EQ("active", s); // no cleanup should have occurred.
}
TEST(FinallyTest, TypeErasedWithoutFactory) {
std::string s = "active";
{
AnyCleanup s_cleaner([&s] { s.append(" clean"); });
EXPECT_EQ("active", s);
}
EXPECT_EQ("active clean", s);
}
struct Appender {
Appender(std::string* s, const std::string& msg) : s_(s), msg_(msg) {}
void operator()() const { s_->append(msg_); }
std::string* s_;
std::string msg_;
};
TEST(CleanupTest, NonLambda) {
std::string s = "active";
{
auto c = gtl::MakeCleanup(Appender(&s, " cleaned"));
AssertTypeEq<decltype(c), gtl::Cleanup<Appender>>();
EXPECT_EQ("active", s);
}
EXPECT_EQ("active cleaned", s);
}
TEST(CleanupTest, Assign) {
std::string s = "0";
{
auto clean1 = gtl::MakeCleanup(Appender(&s, " 1"));
auto clean2 = gtl::MakeCleanup(Appender(&s, " 2"));
EXPECT_EQ("0", s);
clean2 = std::move(clean1);
EXPECT_EQ("0 2", s);
}
EXPECT_EQ("0 2 1", s);
}
TEST(CleanupTest, AssignAny) {
// Check that implicit conversions can happen in assignment.
std::string s = "0";
{
auto clean1 = gtl::MakeCleanup(Appender(&s, " 1"));
AnyCleanup clean2 = gtl::MakeCleanup(Appender(&s, " 2"));
EXPECT_EQ("0", s);
clean2 = std::move(clean1);
EXPECT_EQ("0 2", s);
}
EXPECT_EQ("0 2 1", s);
}
TEST(CleanupTest, AssignFromReleased) {
std::string s = "0";
{
auto clean1 = gtl::MakeCleanup(Appender(&s, " 1"));
auto clean2 = gtl::MakeCleanup(Appender(&s, " 2"));
EXPECT_EQ("0", s);
clean1.release();
clean2 = std::move(clean1);
EXPECT_EQ("0 2", s);
}
EXPECT_EQ("0 2", s);
}
TEST(CleanupTest, AssignToReleased) {
std::string s = "0";
{
auto clean1 = gtl::MakeCleanup(Appender(&s, " 1"));
auto clean2 = gtl::MakeCleanup(Appender(&s, " 2"));
EXPECT_EQ("0", s);
clean2.release();
EXPECT_EQ("0", s);
clean2 = std::move(clean1);
EXPECT_EQ("0", s);
}
EXPECT_EQ("0 1", s);
}
TEST(CleanupTest, AssignToDefaultInitialized) {
std::string s = "0";
{
auto clean1 = gtl::MakeCleanup(Appender(&s, " 1"));
{
AnyCleanup clean2;
EXPECT_EQ("0", s);
clean2 = std::move(clean1);
EXPECT_EQ("0", s);
}
EXPECT_EQ("0 1", s);
}
EXPECT_EQ("0 1", s);
}
class CleanupReferenceTest : public ::testing::Test {
public:
struct F {
int* cp;
int* i;
F(int* cp, int* i) : cp(cp), i(i) {}
F(const F& o) : cp(o.cp), i(o.i) { ++*cp; }
F& operator=(const F& o) {
cp = o.cp;
i = o.i;
++*cp;
return *this;
}
F(F&&) = default;
F& operator=(F&&) = default;
void operator()() const { ++*i; }
};
int copies_ = 0;
int calls_ = 0;
F f_ = F(&copies_, &calls_);
static int g_calls;
void SetUp() override { g_calls = 0; }
static void CleanerFunction() { ++g_calls; }
};
int CleanupReferenceTest::g_calls = 0;
TEST_F(CleanupReferenceTest, FunctionPointer) {
{
auto c = gtl::MakeCleanup(&CleanerFunction);
AssertTypeEq<decltype(c), gtl::Cleanup<void (*)()>>();
EXPECT_EQ(0, g_calls);
}
EXPECT_EQ(1, g_calls);
// Test that a function reference decays to a function pointer.
{
auto c = gtl::MakeCleanup(CleanerFunction);
AssertTypeEq<decltype(c), gtl::Cleanup<void (*)()>>();
EXPECT_EQ(1, g_calls);
}
EXPECT_EQ(2, g_calls);
}
TEST_F(CleanupReferenceTest, AssignLvalue) {
std::string s = "0";
Appender app1(&s, "1");
Appender app2(&s, "2");
{
auto c = gtl::MakeCleanup(app1);
c.release();
c = gtl::MakeCleanup(app2);
EXPECT_EQ("0", s);
app1();
EXPECT_EQ("01", s);
}
EXPECT_EQ("012", s);
}
TEST_F(CleanupReferenceTest, FunctorLvalue) {
// Test that MakeCleanup(lvalue) produces Cleanup<F>, not Cleanup<F&>.
EXPECT_EQ(0, copies_);
EXPECT_EQ(0, calls_);
{
auto c = gtl::MakeCleanup(f_);
AssertTypeEq<decltype(c), gtl::Cleanup<F>>();
EXPECT_EQ(1, copies_);
EXPECT_EQ(0, calls_);
}
EXPECT_EQ(1, copies_);
EXPECT_EQ(1, calls_);
{
auto c = gtl::MakeCleanup(f_);
EXPECT_EQ(2, copies_);
EXPECT_EQ(1, calls_);
F f2 = c.release(); // release is a move.
EXPECT_EQ(2, copies_);
EXPECT_EQ(1, calls_);
auto c2 = gtl::MakeCleanup(f2); // copy
EXPECT_EQ(3, copies_);
EXPECT_EQ(1, calls_);
}
EXPECT_EQ(3, copies_);
EXPECT_EQ(2, calls_);
}
TEST_F(CleanupReferenceTest, FunctorRvalue) {
{
auto c = gtl::MakeCleanup(std::move(f_));
AssertTypeEq<decltype(c), gtl::Cleanup<F>>();
EXPECT_EQ(0, copies_);
EXPECT_EQ(0, calls_);
}
EXPECT_EQ(0, copies_);
EXPECT_EQ(1, calls_);
}
TEST_F(CleanupReferenceTest, FunctorReferenceWrapper) {
{
auto c = gtl::MakeCleanup(std::cref(f_));
AssertTypeEq<decltype(c), gtl::Cleanup<std::reference_wrapper<const F>>>();
EXPECT_EQ(0, copies_);
EXPECT_EQ(0, calls_);
}
EXPECT_EQ(0, copies_);
EXPECT_EQ(1, calls_);
}
volatile int i;
void Incr(volatile int* ip) { ++*ip; }
void Incr() { Incr(&i); }
void BM_Cleanup(::testing::benchmark::State& state) {
for (auto s : state) {
auto fin = gtl::MakeCleanup([] { Incr(); });
}
}
BENCHMARK(BM_Cleanup);
void BM_AnyCleanup(::testing::benchmark::State& state) {
for (auto s : state) {
AnyCleanup fin = gtl::MakeCleanup([] { Incr(); });
}
}
BENCHMARK(BM_AnyCleanup);
void BM_AnyCleanupNoFactory(::testing::benchmark::State& state) {
for (auto s : state) {
AnyCleanup fin([] { Incr(); });
}
}
BENCHMARK(BM_AnyCleanupNoFactory);
void BM_CleanupBound(::testing::benchmark::State& state) {
volatile int* ip = &i;
for (auto s : state) {
auto fin = gtl::MakeCleanup([ip] { Incr(ip); });
}
}
BENCHMARK(BM_CleanupBound);
void BM_AnyCleanupBound(::testing::benchmark::State& state) {
volatile int* ip = &i;
for (auto s : state) {
AnyCleanup fin = gtl::MakeCleanup([ip] { Incr(ip); });
}
}
BENCHMARK(BM_AnyCleanupBound);
void BM_AnyCleanupNoFactoryBound(::testing::benchmark::State& state) {
volatile int* ip = &i;
for (auto s : state) {
AnyCleanup fin([ip] { Incr(ip); });
}
}
BENCHMARK(BM_AnyCleanupNoFactoryBound);
} // namespace
} // namespace tensorflow
tensorflow/core/lib/gtl/compactptrset.h
+29
View File
@@ -0,0 +1,29 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_GTL_COMPACTPTRSET_H_
#define TENSORFLOW_CORE_LIB_GTL_COMPACTPTRSET_H_
#include "xla/tsl/lib/gtl/compactptrset.h"
namespace tensorflow {
namespace gtl {
using ::tsl::gtl::CompactPointerSet; // NOLINT(misc-unused-using-decls)
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_COMPACTPTRSET_H_
tensorflow/core/lib/gtl/edit_distance.h
+108
View File
@@ -0,0 +1,108 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_GTL_EDIT_DISTANCE_H_
#define TENSORFLOW_CORE_LIB_GTL_EDIT_DISTANCE_H_
#include <numeric>
#include "tensorflow/core/lib/gtl/array_slice.h"
#include "tensorflow/core/lib/gtl/inlined_vector.h"
namespace tensorflow {
namespace gtl {
// Calculate the Levenshtein Edit Distance between two contiguous
// sequences, s and t, of type T.
//
// The Levenshtein distance is a symmetric distance defined as the
// smallest number of insertions, deletions, and substitutions
// required to convert sequence s to t (and vice versa).
// Note, this distance does not consider transpositions.
//
// For more details and a reference implementation, see:
// https://en.wikipedia.org/wiki/Levenshtein_distance
//
// This implementation has time complexity O(|s|*|t|)
// and space complexity O(min(|s|, |t|)), where
// |x| := x.size()
//
// A simple call to LevenshteinDistance looks like:
//
// int64 dist = LevenshteinDistance("hi", "bye", std::equal_to<char>());
//
template <typename T, typename Cmp>
inline int64_t LevenshteinDistance(const gtl::ArraySlice<T> s,
const gtl::ArraySlice<T> t, const Cmp& cmp) {
const int64_t s_size = s.size();
const int64_t t_size = t.size();
if (t_size > s_size) return LevenshteinDistance(t, s, cmp);
const T* s_data = s.data();
const T* t_data = t.data();
if (t_size == 0) return s_size;
if (s == t) return 0;
// Create work vector
absl::InlinedVector<int64_t, 32UL> scratch_holder(t_size);
int64_t* scratch = scratch_holder.data();
// Special case for i = 0: Distance between empty string and string
// of length j is just j.
for (size_t j = 1; j < t_size; ++j) scratch[j - 1] = j;
for (size_t i = 1; i <= s_size; ++i) {
// Invariant: scratch[j - 1] equals cost(i - 1, j).
int substitution_base_cost = i - 1;
int insertion_cost = i + 1;
for (size_t j = 1; j <= t_size; ++j) {
// Invariants:
// scratch[k - 1] = cost(i, k) for 0 < k < j.
// scratch[k - 1] = cost(i - 1, k) for j <= k <= t_size.
// substitution_base_cost = cost(i - 1, j - 1)
// insertion_cost = cost(i, j - 1)
const int replacement_cost = cmp(s_data[i - 1], t_data[j - 1]) ? 0 : 1;
const int substitution_cost = substitution_base_cost + replacement_cost;
const int deletion_cost = scratch[j - 1] + 1;
// Select the cheapest edit.
const int cheapest = // = cost(i, j)
std::min(deletion_cost, std::min(insertion_cost, substitution_cost));
// Restore invariant for the next iteration of the loop.
substitution_base_cost = scratch[j - 1]; // = cost(i - 1, j)
scratch[j - 1] = cheapest; // = cost(i, j)
insertion_cost = cheapest + 1; // = cost(i, j) + 1
}
}
return scratch[t_size - 1];
}
template <typename Container1, typename Container2, typename Cmp>
inline int64_t LevenshteinDistance(const Container1& s, const Container2& t,
const Cmp& cmp) {
return LevenshteinDistance(
gtl::ArraySlice<typename Container1::value_type>(s.data(), s.size()),
gtl::ArraySlice<typename Container1::value_type>(t.data(), t.size()),
cmp);
}
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_EDIT_DISTANCE_H_
tensorflow/core/lib/gtl/edit_distance_test.cc
+154
View File
@@ -0,0 +1,154 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/lib/gtl/edit_distance.h"
#include <cctype>
#include <vector>
#include "absl/strings/ascii.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/platform/test_benchmark.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace gtl {
namespace {
class LevenshteinDistanceTest : public ::testing::Test {
protected:
std::vector<char> empty_;
std::string s1_;
std::string s1234_;
std::string s567_;
std::string kilo_;
std::string kilogram_;
std::string mother_;
std::string grandmother_;
std::string lower_;
std::string upper_;
std::vector<char> ebab_;
std::vector<char> abcd_;
void SetUp() override {
s1_ = "1";
s1234_ = "1234";
s567_ = "567";
kilo_ = "kilo";
kilogram_ = "kilogram";
mother_ = "mother";
grandmother_ = "grandmother";
lower_ = "lower case";
upper_ = "UPPER case";
ebab_ = {'e', 'b', 'a', 'b'};
abcd_ = {'a', 'b', 'c', 'd'};
}
};
TEST_F(LevenshteinDistanceTest, BothEmpty) {
ASSERT_EQ(LevenshteinDistance(empty_, empty_, std::equal_to<char>()), 0);
}
TEST_F(LevenshteinDistanceTest, Symmetry) {
ASSERT_EQ(LevenshteinDistance(ebab_, abcd_, std::equal_to<char>()), 3);
ASSERT_EQ(LevenshteinDistance(abcd_, ebab_, std::equal_to<char>()), 3);
}
TEST_F(LevenshteinDistanceTest, OneEmpty) {
ASSERT_EQ(LevenshteinDistance(s1234_, empty_, std::equal_to<char>()), 4);
ASSERT_EQ(LevenshteinDistance(empty_, s567_, std::equal_to<char>()), 3);
}
TEST_F(LevenshteinDistanceTest, SingleElement) {
ASSERT_EQ(LevenshteinDistance(s1234_, s1_, std::equal_to<char>()), 3);
ASSERT_EQ(LevenshteinDistance(s1_, s1234_, std::equal_to<char>()), 3);
}
TEST_F(LevenshteinDistanceTest, Prefix) {
ASSERT_EQ(LevenshteinDistance(kilo_, kilogram_, std::equal_to<char>()), 4);
ASSERT_EQ(LevenshteinDistance(kilogram_, kilo_, std::equal_to<char>()), 4);
}
TEST_F(LevenshteinDistanceTest, Suffix) {
ASSERT_EQ(LevenshteinDistance(mother_, grandmother_, std::equal_to<char>()),
5);
ASSERT_EQ(LevenshteinDistance(grandmother_, mother_, std::equal_to<char>()),
5);
}
TEST_F(LevenshteinDistanceTest, DifferentComparisons) {
ASSERT_EQ(LevenshteinDistance(lower_, upper_, std::equal_to<char>()), 5);
ASSERT_EQ(LevenshteinDistance(upper_, lower_, std::equal_to<char>()), 5);
ASSERT_EQ(
LevenshteinDistance(absl::Span<const char>(lower_.data(), lower_.size()),
absl::Span<const char>(upper_.data(), upper_.size()),
std::equal_to<char>()),
5);
auto no_case_cmp = [](char c1, char c2) {
return absl::ascii_tolower(c1) == absl::ascii_tolower(c2);
};
ASSERT_EQ(LevenshteinDistance(lower_, upper_, no_case_cmp), 3);
ASSERT_EQ(LevenshteinDistance(upper_, lower_, no_case_cmp), 3);
}
TEST_F(LevenshteinDistanceTest, Vectors) {
ASSERT_EQ(
LevenshteinDistance(std::string("algorithm"), std::string("altruistic"),
std::equal_to<char>()),
6);
}
static void BM_EditDistanceHelper(::testing::benchmark::State& state, int len,
bool completely_different) {
std::string a =
"The quick brown fox jumped over the lazy dog and on and on and on"
" Every good boy deserves fudge. In fact, this is a very long sentence "
" w/many bytes..";
while (a.size() < static_cast<size_t>(len)) {
a = a + a;
}
std::string b = a;
if (completely_different) {
for (size_t i = 0; i < b.size(); i++) {
b[i]++;
}
}
for (auto s : state) {
LevenshteinDistance(absl::Span<const char>(a.data(), len),
absl::Span<const char>(b.data(), len),
std::equal_to<char>());
}
}
static void BM_EditDistanceSame(::testing::benchmark::State& state) {
BM_EditDistanceHelper(state, state.range(0), false);
}
static void BM_EditDistanceDiff(::testing::benchmark::State& state) {
BM_EditDistanceHelper(state, state.range(0), true);
}
BENCHMARK(BM_EditDistanceSame)->Arg(5);
BENCHMARK(BM_EditDistanceSame)->Arg(50);
BENCHMARK(BM_EditDistanceSame)->Arg(200);
BENCHMARK(BM_EditDistanceSame)->Arg(1000);
BENCHMARK(BM_EditDistanceDiff)->Arg(5);
BENCHMARK(BM_EditDistanceDiff)->Arg(50);
BENCHMARK(BM_EditDistanceDiff)->Arg(200);
BENCHMARK(BM_EditDistanceDiff)->Arg(1000);
} // namespace
} // namespace gtl
} // namespace tensorflow
tensorflow/core/lib/gtl/flatmap.h
+33
View File
@@ -0,0 +1,33 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_GTL_FLATMAP_H_
#define TENSORFLOW_CORE_LIB_GTL_FLATMAP_H_
#include "xla/tsl/lib/gtl/flatmap.h"
#include "tensorflow/core/lib/gtl/flatrep.h"
#include "tensorflow/core/lib/hash/hash.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace gtl {
using tsl::gtl::FlatMap; // NOLINT(misc-unused-using-decls)
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_FLATMAP_H_
tensorflow/core/lib/gtl/flatrep.h
+31
View File
@@ -0,0 +1,31 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_GTL_FLATREP_H_
#define TENSORFLOW_CORE_LIB_GTL_FLATREP_H_
#include "xla/tsl/lib/gtl/flatrep.h"
namespace tensorflow {
namespace gtl {
namespace internal {
using tsl::gtl::internal::FlatRep; // NOLINT(misc-unused-using-decls)
} // namespace internal
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_FLATREP_H_
tensorflow/core/lib/gtl/flatset.h
+29
View File
@@ -0,0 +1,29 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_GTL_FLATSET_H_
#define TENSORFLOW_CORE_LIB_GTL_FLATSET_H_
#include "xla/tsl/lib/gtl/flatset.h"
namespace tensorflow {
namespace gtl {
using tsl::gtl::FlatSet; // NOLINT(misc-unused-using-decls)
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_FLATSET_H_
tensorflow/core/lib/gtl/inlined_vector.h
+33
View File
@@ -0,0 +1,33 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_GTL_INLINED_VECTOR_H_
#define TENSORFLOW_CORE_LIB_GTL_INLINED_VECTOR_H_
#include "xla/tsl/lib/gtl/inlined_vector.h" // IWYU pragma: export
// TODO(kramerb): This is kept only because lots of targets transitively depend
// on it. Remove all targets' dependencies.
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace gtl {
using ::tsl::gtl::InlinedVector; // NOLINT(misc-unused-using-decls)
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_INLINED_VECTOR_H_
tensorflow/core/lib/gtl/int_type.h
+30
View File
@@ -0,0 +1,30 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_GTL_INT_TYPE_H_
#define TENSORFLOW_CORE_LIB_GTL_INT_TYPE_H_
#include "xla/tsl/lib/gtl/int_type.h"
namespace tensorflow {
namespace gtl {
using ::tsl::gtl::IntType; // NOLINT(misc-unused-using-decls)
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_INT_TYPE_H_
tensorflow/core/lib/gtl/iterator_range.h
+39
View File
@@ -0,0 +1,39 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This provides a very simple, boring adaptor for a begin and end iterator
// into a range type. This should be used to build range views that work well
// with range based for loops and range based constructors.
//
// Note that code here follows more standards-based coding conventions as it
// is mirroring proposed interfaces for standardization.
//
// Converted from chandlerc@'s code to Google style by joshl@.
#ifndef TENSORFLOW_CORE_LIB_GTL_ITERATOR_RANGE_H_
#define TENSORFLOW_CORE_LIB_GTL_ITERATOR_RANGE_H_
#include "xla/tsl/lib/gtl/iterator_range.h"
namespace tensorflow {
namespace gtl {
// NOLINTBEGIN(misc-unused-using-decls)
using ::tsl::gtl::iterator_range;
using ::tsl::gtl::make_range;
// NOLINTEND(misc-unused-using-decls)
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_ITERATOR_RANGE_H_
tensorflow/core/lib/gtl/manual_constructor.h
+246
View File
@@ -0,0 +1,246 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// ManualConstructor statically-allocates space in which to store some
// object, but does not initialize it. You can then call the constructor
// and destructor for the object yourself as you see fit. This is useful
// for memory management optimizations, where you want to initialize and
// destroy an object multiple times but only allocate it once.
//
// (When I say ManualConstructor statically allocates space, I mean that
// the ManualConstructor object itself is forced to be the right size.)
#ifndef TENSORFLOW_CORE_LIB_GTL_MANUAL_CONSTRUCTOR_H_
#define TENSORFLOW_CORE_LIB_GTL_MANUAL_CONSTRUCTOR_H_
#include <stddef.h>
#include <new>
#include <utility>
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mem.h"
namespace tensorflow {
namespace gtl {
namespace internal {
//
// Provides a char array with the exact same alignment as another type. The
// first parameter must be a complete type, the second parameter is how many
// of that type to provide space for.
//
// TF_LIB_GTL_ALIGNED_CHAR_ARRAY(struct stat, 16) storage_;
//
// Because MSVC and older GCCs require that the argument to their alignment
// construct to be a literal constant integer, we use a template instantiated
// at all the possible powers of two.
#ifndef SWIG
template <int alignment, int size>
struct AlignType {};
template <int size>
struct AlignType<0, size> {
typedef char result[size];
};
#if defined(_MSC_VER)
#define TF_LIB_GTL_ALIGN_ATTRIBUTE(X) __declspec(align(X))
#define TF_LIB_GTL_ALIGN_OF(T) __alignof(T)
#else
#define TF_LIB_GTL_ALIGN_ATTRIBUTE(X) __attribute__((aligned(X)))
#define TF_LIB_GTL_ALIGN_OF(T) __alignof__(T)
#endif
#if defined(TF_LIB_GTL_ALIGN_ATTRIBUTE)
#define TF_LIB_GTL_ALIGNTYPE_TEMPLATE(X) \
template <int size> \
struct AlignType<X, size> { \
typedef TF_LIB_GTL_ALIGN_ATTRIBUTE(X) char result[size]; \
}
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(1);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(2);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(4);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(8);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(16);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(32);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(64);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(128);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(256);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(512);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(1024);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(2048);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(4096);
TF_LIB_GTL_ALIGNTYPE_TEMPLATE(8192);
// Any larger and MSVC++ will complain.
#define TF_LIB_GTL_ALIGNED_CHAR_ARRAY(T, Size) \
typename tensorflow::gtl::internal::AlignType<TF_LIB_GTL_ALIGN_OF(T), \
sizeof(T) * Size>::result
#undef TF_LIB_GTL_ALIGNTYPE_TEMPLATE
#undef TF_LIB_GTL_ALIGN_ATTRIBUTE
#else // defined(TF_LIB_GTL_ALIGN_ATTRIBUTE)
#error "You must define TF_LIB_GTL_ALIGNED_CHAR_ARRAY for your compiler."
#endif // defined(TF_LIB_GTL_ALIGN_ATTRIBUTE)
#else // !SWIG
// SWIG can't represent alignment and doesn't care about alignment on data
// members (it works fine without it).
template <typename Size>
struct AlignType {
typedef char result[Size];
};
#define TF_LIB_GTL_ALIGNED_CHAR_ARRAY(T, Size) \
tensorflow::gtl::internal::AlignType<Size * sizeof(T)>::result
// Enough to parse with SWIG, will never be used by running code.
#define TF_LIB_GTL_ALIGN_OF(Type) 16
#endif // !SWIG
} // namespace internal
} // namespace gtl
template <typename Type>
class ManualConstructor {
public:
// No constructor or destructor because one of the most useful uses of
// this class is as part of a union, and members of a union cannot have
// constructors or destructors. And, anyway, the whole point of this
// class is to bypass these.
// Support users creating arrays of ManualConstructor<>s. This ensures that
// the array itself has the correct alignment.
static void* operator new[](size_t size) {
return tsl::port::AlignedMalloc(
size, static_cast<std::align_val_t>(TF_LIB_GTL_ALIGN_OF(Type)));
}
static void operator delete[](void* mem) { port::AlignedFree(mem); }
inline Type* get() { return reinterpret_cast<Type*>(space_); }
inline const Type* get() const {
return reinterpret_cast<const Type*>(space_);
}
inline Type* operator->() { return get(); }
inline const Type* operator->() const { return get(); }
inline Type& operator*() { return *get(); }
inline const Type& operator*() const { return *get(); }
inline void Init() { new (space_) Type; }
// Init() constructs the Type instance using the given arguments
// (which are forwarded to Type's constructor). In C++11, Init() can
// take any number of arguments of any type, and forwards them perfectly.
// On pre-C++11 platforms, it can take up to 11 arguments, and may not be
// able to forward certain kinds of arguments.
//
// Note that Init() with no arguments performs default-initialization,
// not zero-initialization (i.e it behaves the same as "new Type;", not
// "new Type();"), so it will leave non-class types uninitialized.
#ifdef LANG_CXX11
template <typename... Ts>
inline void Init(Ts&&... args) { // NOLINT
new (space_) Type(std::forward<Ts>(args)...); // NOLINT
}
#else // !defined(LANG_CXX11)
template <typename T1>
inline void Init(const T1& p1) {
new (space_) Type(p1);
}
template <typename T1, typename T2>
inline void Init(const T1& p1, const T2& p2) {
new (space_) Type(p1, p2);
}
template <typename T1, typename T2, typename T3>
inline void Init(const T1& p1, const T2& p2, const T3& p3) {
new (space_) Type(p1, p2, p3);
}
template <typename T1, typename T2, typename T3, typename T4>
inline void Init(const T1& p1, const T2& p2, const T3& p3, const T4& p4) {
new (space_) Type(p1, p2, p3, p4);
}
template <typename T1, typename T2, typename T3, typename T4, typename T5>
inline void Init(const T1& p1, const T2& p2, const T3& p3, const T4& p4,
const T5& p5) {
new (space_) Type(p1, p2, p3, p4, p5);
}
template <typename T1, typename T2, typename T3, typename T4, typename T5,
typename T6>
inline void Init(const T1& p1, const T2& p2, const T3& p3, const T4& p4,
const T5& p5, const T6& p6) {
new (space_) Type(p1, p2, p3, p4, p5, p6);
}
template <typename T1, typename T2, typename T3, typename T4, typename T5,
typename T6, typename T7>
inline void Init(const T1& p1, const T2& p2, const T3& p3, const T4& p4,
const T5& p5, const T6& p6, const T7& p7) {
new (space_) Type(p1, p2, p3, p4, p5, p6, p7);
}
template <typename T1, typename T2, typename T3, typename T4, typename T5,
typename T6, typename T7, typename T8>
inline void Init(const T1& p1, const T2& p2, const T3& p3, const T4& p4,
const T5& p5, const T6& p6, const T7& p7, const T8& p8) {
new (space_) Type(p1, p2, p3, p4, p5, p6, p7, p8);
}
template <typename T1, typename T2, typename T3, typename T4, typename T5,
typename T6, typename T7, typename T8, typename T9>
inline void Init(const T1& p1, const T2& p2, const T3& p3, const T4& p4,
const T5& p5, const T6& p6, const T7& p7, const T8& p8,
const T9& p9) {
new (space_) Type(p1, p2, p3, p4, p5, p6, p7, p8, p9);
}
template <typename T1, typename T2, typename T3, typename T4, typename T5,
typename T6, typename T7, typename T8, typename T9, typename T10>
inline void Init(const T1& p1, const T2& p2, const T3& p3, const T4& p4,
const T5& p5, const T6& p6, const T7& p7, const T8& p8,
const T9& p9, const T10& p10) {
new (space_) Type(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10);
}
template <typename T1, typename T2, typename T3, typename T4, typename T5,
typename T6, typename T7, typename T8, typename T9, typename T10,
typename T11>
inline void Init(const T1& p1, const T2& p2, const T3& p3, const T4& p4,
const T5& p5, const T6& p6, const T7& p7, const T8& p8,
const T9& p9, const T10& p10, const T11& p11) {
new (space_) Type(p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11);
}
#endif // LANG_CXX11
inline void Destroy() { get()->~Type(); }
private:
TF_LIB_GTL_ALIGNED_CHAR_ARRAY(Type, 1) space_;
};
#undef TF_LIB_GTL_ALIGNED_CHAR_ARRAY
#undef TF_LIB_GTL_ALIGN_OF
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_MANUAL_CONSTRUCTOR_H_
tensorflow/core/lib/gtl/manual_constructor_test.cc
+124
View File
@@ -0,0 +1,124 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/lib/gtl/manual_constructor.h"
#include <stdint.h>
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/test.h"
namespace tensorflow {
namespace {
static int constructor_count_ = 0;
template <int kSize>
struct TestN {
TestN() { ++constructor_count_; }
~TestN() { --constructor_count_; }
char a[kSize];
};
typedef TestN<1> Test1;
typedef TestN<2> Test2;
typedef TestN<3> Test3;
typedef TestN<4> Test4;
typedef TestN<5> Test5;
typedef TestN<9> Test9;
typedef TestN<15> Test15;
} // namespace
namespace {
TEST(ManualConstructorTest, Sizeof) {
CHECK_EQ(sizeof(ManualConstructor<Test1>), sizeof(Test1));
CHECK_EQ(sizeof(ManualConstructor<Test2>), sizeof(Test2));
CHECK_EQ(sizeof(ManualConstructor<Test3>), sizeof(Test3));
CHECK_EQ(sizeof(ManualConstructor<Test4>), sizeof(Test4));
CHECK_EQ(sizeof(ManualConstructor<Test5>), sizeof(Test5));
CHECK_EQ(sizeof(ManualConstructor<Test9>), sizeof(Test9));
CHECK_EQ(sizeof(ManualConstructor<Test15>), sizeof(Test15));
CHECK_EQ(constructor_count_, 0);
ManualConstructor<Test1> mt[4];
CHECK_EQ(sizeof(mt), 4);
CHECK_EQ(constructor_count_, 0);
mt[0].Init();
CHECK_EQ(constructor_count_, 1);
mt[0].Destroy();
}
TEST(ManualConstructorTest, Alignment) {
// We want to make sure that ManualConstructor aligns its memory properly
// on a word barrier. Otherwise, it might be unexpectedly slow, since
// memory access will be unaligned.
struct {
char a;
ManualConstructor<void*> b;
} test1;
struct {
char a;
void* b;
} control1;
// TODO(bww): Make these tests more direct with C++11 alignment_of<T>::value.
EXPECT_EQ(reinterpret_cast<char*>(test1.b.get()) - &test1.a,
reinterpret_cast<char*>(&control1.b) - &control1.a);
EXPECT_EQ(reinterpret_cast<intptr_t>(test1.b.get()) % sizeof(control1.b), 0);
struct {
char a;
ManualConstructor<long double> b;
} test2;
struct {
char a;
long double b;
} control2;
EXPECT_EQ(reinterpret_cast<char*>(test2.b.get()) - &test2.a,
reinterpret_cast<char*>(&control2.b) - &control2.a);
EXPECT_EQ(reinterpret_cast<intptr_t>(test2.b.get()) % alignof(long double),
0);
}
TEST(ManualConstructorTest, DefaultInitialize) {
struct X {
X() : x(123) {}
int x;
};
union {
ManualConstructor<X> x;
ManualConstructor<int> y;
} u;
*u.y = -1;
u.x.Init(); // should default-initialize u.x
EXPECT_EQ(123, u.x->x);
}
TEST(ManualConstructorTest, ZeroInitializePOD) {
union {
ManualConstructor<int> x;
ManualConstructor<int> y;
} u;
*u.y = -1;
u.x.Init(); // should not zero-initialize u.x
EXPECT_EQ(-1, *u.y);
}
} // namespace
} // namespace tensorflow
tensorflow/core/lib/gtl/map_util.h
+40
View File
@@ -0,0 +1,40 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This file provides utility functions for use with STL map-like data
// structures, such as std::map and hash_map. Some functions will also work with
// sets, such as ContainsKey().
#ifndef TENSORFLOW_CORE_LIB_GTL_MAP_UTIL_H_
#define TENSORFLOW_CORE_LIB_GTL_MAP_UTIL_H_
#include "xla/tsl/lib/gtl/map_util.h"
namespace tensorflow {
namespace gtl {
// NOLINTBEGIN(misc-unused-using-decls)
using ::tsl::gtl::EraseKeyReturnValuePtr;
using ::tsl::gtl::FindOrNull;
using ::tsl::gtl::FindPtrOrNull;
using ::tsl::gtl::FindWithDefault;
using ::tsl::gtl::InsertIfNotPresent;
using ::tsl::gtl::InsertOrUpdate;
using ::tsl::gtl::LookupOrInsert;
using ::tsl::gtl::ReverseMap;
// NOLINTEND(misc-unused-using-decls)
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_MAP_UTIL_H_
tensorflow/core/lib/gtl/priority_queue_util.h
+55
View File
@@ -0,0 +1,55 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_GTL_PRIORITY_QUEUE_UTIL_H_
#define TENSORFLOW_CORE_LIB_GTL_PRIORITY_QUEUE_UTIL_H_
#include <algorithm>
#include <queue>
#include <utility>
namespace tensorflow {
namespace gtl {
// Removes the top element from a std::priority_queue and returns it.
// Supports movable types.
template <typename T, typename Container, typename Comparator>
T ConsumeTop(std::priority_queue<T, Container, Comparator>* q) {
// std::priority_queue is required to implement pop() as if it
// called:
// std::pop_heap()
// c.pop_back()
// unfortunately, it does not provide access to the removed element.
// If the element is move only (such as a unique_ptr), there is no way to
// reclaim it in the standard API. std::priority_queue does, however, expose
// the underlying container as a protected member, so we use that access
// to extract the desired element between those two calls.
using Q = std::priority_queue<T, Container, Comparator>;
struct Expose : Q {
using Q::c;
using Q::comp;
};
auto& c = q->*&Expose::c;
auto& comp = q->*&Expose::comp;
std::pop_heap(c.begin(), c.end(), comp);
auto r = std::move(c.back());
c.pop_back();
return r;
}
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_PRIORITY_QUEUE_UTIL_H_
tensorflow/core/lib/gtl/subtle/BUILD
+18
View File
@@ -0,0 +1,18 @@
# Description:
# gtl subtle packages.
load("//tensorflow:tensorflow.default.bzl", "filegroup")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
filegroup(
name = "map_traits",
srcs = [
"map_traits.h",
"@xla//xla/tsl/lib/gtl/subtle:map_traits",
],
visibility = ["//tensorflow/core/lib/gtl:__pkg__"],
)
tensorflow/core/lib/gtl/subtle/map_traits.h
+44
View File
@@ -0,0 +1,44 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Traits classes for performing uniform lookup on different map value types.
//
// The access is computed as follows:
//
// 1. If T has a `first` or `second` field, use them.
// 2. Otherwise if it has `key()` or `value()` methods, use them.
// 3. Otherwise the program is ill-formed.
#ifndef TENSORFLOW_CORE_LIB_GTL_SUBTLE_MAP_TRAITS_H_
#define TENSORFLOW_CORE_LIB_GTL_SUBTLE_MAP_TRAITS_H_
#include "xla/tsl/lib/gtl/subtle/map_traits.h"
namespace tensorflow {
namespace gtl {
namespace subtle {
namespace internal_map_traits {
// NOLINTBEGIN(misc-unused-using-decls)
using ::tsl::gtl::subtle::internal_map_traits::GetKey;
using ::tsl::gtl::subtle::internal_map_traits::GetMapped;
using ::tsl::gtl::subtle::internal_map_traits::Rank0;
using ::tsl::gtl::subtle::internal_map_traits::Rank1;
// NOLINTEND(misc-unused-using-decls)
} // namespace internal_map_traits
} // namespace subtle
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_SUBTLE_MAP_TRAITS_H_
tensorflow/core/lib/gtl/top_n.h
+336
View File
@@ -0,0 +1,336 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This simple class finds the top n elements of an incrementally provided set
// of elements which you push one at a time. If the number of elements exceeds
// n, the lowest elements are incrementally dropped. At the end you get
// a vector of the top elements sorted in descending order (through Extract() or
// ExtractNondestructive()), or a vector of the top elements but not sorted
// (through ExtractUnsorted() or ExtractUnsortedNondestructive()).
//
// The value n is specified in the constructor. If there are p elements pushed
// altogether:
// The total storage requirements are O(min(n, p)) elements
// The running time is O(p * log(min(n, p))) comparisons
// If n is a constant, the total storage required is a constant and the running
// time is linear in p.
//
// NOTE(zhifengc): There is a way to do this in O(min(n, p)) storage and O(p)
// runtime. The basic idea is to repeatedly fill up a buffer of 2 * n elements,
// discarding the lowest n elements whenever the buffer is full using a linear-
// time median algorithm. This may have better performance when the input
// sequence is partially sorted.
//
// NOTE(zhifengc): This class should be redesigned to avoid reallocating a
// vector for each Extract.
#ifndef TENSORFLOW_CORE_LIB_GTL_TOP_N_H_
#define TENSORFLOW_CORE_LIB_GTL_TOP_N_H_
#include <stddef.h>
#include <algorithm>
#include <functional>
#include <string>
#include <vector>
#include "tensorflow/core/platform/logging.h"
namespace tensorflow {
namespace gtl {
// Cmp is an stl binary predicate. Note that Cmp is the "greater" predicate,
// not the more commonly used "less" predicate.
//
// If you use a "less" predicate here, the TopN will pick out the bottom N
// elements out of the ones passed to it, and it will return them sorted in
// ascending order.
//
// TopN is rule-of-zero copyable and movable if its members are.
template <class T, class Cmp = std::greater<T> >
class TopN {
public:
// The TopN is in one of the three states:
//
// o UNORDERED: this is the state an instance is originally in,
// where the elements are completely orderless.
//
// o BOTTOM_KNOWN: in this state, we keep the invariant that there
// is at least one element in it, and the lowest element is at
// position 0. The elements in other positions remain
// unsorted. This state is reached if the state was originally
// UNORDERED and a peek_bottom() function call is invoked.
//
// o HEAP_SORTED: in this state, the array is kept as a heap and
// there are exactly limit_ elements in the array. This
// state is reached when at least (limit_+1) elements are
// pushed in.
//
// The state transition graph is at follows:
//
// peek_bottom() (limit_+1) elements pushed
// UNORDERED --------------> BOTTOM_KNOWN --------------------> HEAP_SORTED
// | ^
// | (limit_+1) elements pushed |
// +-----------------------------------------------------------+
enum State { UNORDERED, BOTTOM_KNOWN, HEAP_SORTED };
using UnsortedIterator = typename std::vector<T>::const_iterator;
// 'limit' is the maximum number of top results to return.
explicit TopN(size_t limit) : TopN(limit, Cmp()) {}
TopN(size_t limit, const Cmp &cmp) : limit_(limit), cmp_(cmp) {}
size_t limit() const { return limit_; }
// Number of elements currently held by this TopN object. This
// will be no greater than 'limit' passed to the constructor.
size_t size() const { return elements_.size(); }
bool empty() const { return size() == 0; }
// If you know how many elements you will push at the time you create the
// TopN object, you can call reserve to preallocate the memory that TopN
// will need to process all 'n' pushes. Calling this method is optional.
void reserve(size_t n) {
// We may need limit_+1 for the case where we transition from an unsorted
// set of limit_ elements to a heap.
elements_.reserve(std::min(n, limit_ + 1));
}
// Push 'v'. If the maximum number of elements was exceeded, drop the
// lowest element and return it in 'dropped' (if given). If the maximum is not
// exceeded, 'dropped' will remain unchanged. 'dropped' may be omitted or
// nullptr, in which case it is not filled in.
// Requires: T is CopyAssignable, Swappable
void push(const T &v) { push(v, nullptr); }
void push(const T &v, T *dropped) { PushInternal(v, dropped); }
// Move overloads of push.
// Requires: T is MoveAssignable, Swappable
void push(T &&v) { // NOLINT(build/c++11)
push(std::move(v), nullptr);
}
void push(T &&v, T *dropped) { // NOLINT(build/c++11)
PushInternal(std::move(v), dropped);
}
// Peeks the bottom result without calling Extract()
const T &peek_bottom();
// Extract the elements as a vector sorted in descending order. The caller
// assumes ownership of the vector and must delete it when done. This is a
// destructive operation. The only method that can be called immediately
// after Extract() is Reset().
std::vector<T> *Extract();
// Similar to Extract(), but makes no guarantees the elements are in sorted
// order. As with Extract(), the caller assumes ownership of the vector and
// must delete it when done. This is a destructive operation. The only
// method that can be called immediately after ExtractUnsorted() is Reset().
std::vector<T> *ExtractUnsorted();
// A non-destructive version of Extract(). Copy the elements in a new vector
// sorted in descending order and return it. The caller assumes ownership of
// the new vector and must delete it when done. After calling
// ExtractNondestructive(), the caller can continue to push() new elements.
std::vector<T> *ExtractNondestructive() const;
// A non-destructive version of Extract(). Copy the elements to a given
// vector sorted in descending order. After calling
// ExtractNondestructive(), the caller can continue to push() new elements.
// Note:
// 1. The given argument must to be allocated.
// 2. Any data contained in the vector prior to the call will be deleted
// from it. After the call the vector will contain only the elements
// from the data structure.
void ExtractNondestructive(std::vector<T> *output) const;
// A non-destructive version of ExtractUnsorted(). Copy the elements in a new
// vector and return it, with no guarantees the elements are in sorted order.
// The caller assumes ownership of the new vector and must delete it when
// done. After calling ExtractUnsortedNondestructive(), the caller can
// continue to push() new elements.
std::vector<T> *ExtractUnsortedNondestructive() const;
// A non-destructive version of ExtractUnsorted(). Copy the elements into
// a given vector, with no guarantees the elements are in sorted order.
// After calling ExtractUnsortedNondestructive(), the caller can continue
// to push() new elements.
// Note:
// 1. The given argument must to be allocated.
// 2. Any data contained in the vector prior to the call will be deleted
// from it. After the call the vector will contain only the elements
// from the data structure.
void ExtractUnsortedNondestructive(std::vector<T> *output) const;
// Return an iterator to the beginning (end) of the container,
// with no guarantees about the order of iteration. These iterators are
// invalidated by mutation of the data structure.
UnsortedIterator unsorted_begin() const { return elements_.begin(); }
UnsortedIterator unsorted_end() const { return elements_.end(); }
// Accessor for comparator template argument.
Cmp *comparator() { return &cmp_; }
// This removes all elements. If Extract() or ExtractUnsorted() have been
// called, this will put it back in an empty but useable state.
void Reset();
private:
template <typename U>
void PushInternal(U &&v, T *dropped); // NOLINT(build/c++11)
// elements_ can be in one of two states:
// elements_.size() <= limit_ && state_ != HEAP_SORTED:
// elements_ is an unsorted vector of elements pushed so far.
// elements_.size() == limit_ && state_ == HEAP_SORTED:
// elements_ is an stl heap.
std::vector<T> elements_;
size_t limit_; // Maximum number of elements to find
Cmp cmp_; // Greater-than comparison function
State state_ = UNORDERED;
};
// ----------------------------------------------------------------------
// Implementations of non-inline functions
template <class T, class Cmp>
template <typename U>
void TopN<T, Cmp>::PushInternal(U &&v, T *dropped) { // NOLINT(build/c++11)
if (limit_ == 0) {
if (dropped) *dropped = std::forward<U>(v); // NOLINT(build/c++11)
return;
}
if (state_ != HEAP_SORTED) {
// We may temporarily extend one beyond limit_ elements here. This is
// necessary for finding and removing the smallest element.
elements_.push_back(std::forward<U>(v)); // NOLINT(build/c++11)
if (elements_.size() == limit_ + 1) {
// Transition from unsorted vector to a heap.
std::make_heap(elements_.begin(), elements_.end(), cmp_);
std::pop_heap(elements_.begin(), elements_.end(), cmp_);
if (dropped) *dropped = std::move(elements_.back());
elements_.pop_back(); // Restore to size limit_.
state_ = HEAP_SORTED;
} else if (state_ == UNORDERED ||
cmp_(elements_.back(), elements_.front())) {
// Easy case: we just push the new element back
} else {
// To maintain the BOTTOM_KNOWN state, we need to make sure that
// the element at position 0 is always the smallest. So we put
// the new element at position 0 and push the original bottom
// element in the back.
// Warning: this code is subtle.
using std::swap;
swap(elements_.front(), elements_.back());
}
} else {
// Only insert the new element if it is greater than the least element.
if (cmp_(v, elements_.front())) {
// Remove the top (smallest) element of the min heap, then push the new
// value in.
std::pop_heap(elements_.begin(), elements_.end(), cmp_);
if (dropped) *dropped = std::move(elements_.back());
elements_.back() = std::forward<U>(v);
std::push_heap(elements_.begin(), elements_.end(), cmp_);
} else {
if (dropped) *dropped = std::forward<U>(v); // NOLINT(build/c++11)
}
}
}
template <class T, class Cmp>
const T &TopN<T, Cmp>::peek_bottom() {
CHECK(!empty());
if (state_ == UNORDERED) {
// We need to do a linear scan to find out the bottom element
int min_candidate = 0;
for (size_t i = 1; i < elements_.size(); ++i) {
if (cmp_(elements_[min_candidate], elements_[i])) {
min_candidate = i;
}
}
// By swapping the element at position 0 and the minimal
// element, we transition to the BOTTOM_KNOWN state
if (min_candidate != 0) {
using std::swap;
swap(elements_[0], elements_[min_candidate]);
}
state_ = BOTTOM_KNOWN;
}
return elements_.front();
}
template <class T, class Cmp>
std::vector<T> *TopN<T, Cmp>::Extract() {
auto out = new std::vector<T>;
out->swap(elements_);
if (state_ != HEAP_SORTED) {
std::sort(out->begin(), out->end(), cmp_);
} else {
std::sort_heap(out->begin(), out->end(), cmp_);
}
return out;
}
template <class T, class Cmp>
std::vector<T> *TopN<T, Cmp>::ExtractUnsorted() {
auto out = new std::vector<T>;
out->swap(elements_);
return out;
}
template <class T, class Cmp>
std::vector<T> *TopN<T, Cmp>::ExtractNondestructive() const {
auto out = new std::vector<T>;
ExtractNondestructive(out);
return out;
}
template <class T, class Cmp>
void TopN<T, Cmp>::ExtractNondestructive(std::vector<T> *output) const {
CHECK(output);
*output = elements_;
if (state_ != HEAP_SORTED) {
std::sort(output->begin(), output->end(), cmp_);
} else {
std::sort_heap(output->begin(), output->end(), cmp_);
}
}
template <class T, class Cmp>
std::vector<T> *TopN<T, Cmp>::ExtractUnsortedNondestructive() const {
auto elements = new std::vector<T>;
ExtractUnsortedNondestructive(elements);
return elements;
}
template <class T, class Cmp>
void TopN<T, Cmp>::ExtractUnsortedNondestructive(std::vector<T> *output) const {
CHECK(output);
*output = elements_;
}
template <class T, class Cmp>
void TopN<T, Cmp>::Reset() {
elements_.clear();
state_ = UNORDERED;
}
} // namespace gtl
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_GTL_TOP_N_H_
tensorflow/core/lib/gtl/top_n_test.cc
+266
View File
@@ -0,0 +1,266 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Unit test for TopN.
#include "tensorflow/core/lib/gtl/top_n.h"
#include <string>
#include <vector>
#include "tensorflow/core/lib/random/simple_philox.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/platform/types.h"
namespace {
using tensorflow::string;
using tensorflow::gtl::TopN;
using tensorflow::random::PhiloxRandom;
using tensorflow::random::SimplePhilox;
// Move the contents from an owned raw pointer, returning by value.
// Objects are easier to manage by value.
template <class T>
T ConsumeRawPtr(T *p) {
T tmp = std::move(*p);
delete p;
return tmp;
}
template <class Cmp>
void TestIntTopNHelper(size_t limit, size_t n_elements, const Cmp &cmp,
SimplePhilox *random, bool test_peek,
bool test_extract_unsorted) {
LOG(INFO) << "Testing limit=" << limit << ", n_elements=" << n_elements
<< ", test_peek=" << test_peek
<< ", test_extract_unsorted=" << test_extract_unsorted;
TopN<int, Cmp> top(limit, cmp);
std::vector<int> shadow(n_elements);
for (int i = 0; i != n_elements; ++i) shadow[i] = random->Uniform(limit);
for (int e : shadow) top.push(e);
std::sort(shadow.begin(), shadow.end(), cmp);
size_t top_size = std::min(limit, n_elements);
EXPECT_EQ(top_size, top.size());
if (test_peek && top_size != 0) {
EXPECT_EQ(shadow[top_size - 1], top.peek_bottom());
}
std::vector<int> v;
if (test_extract_unsorted) {
v = ConsumeRawPtr(top.ExtractUnsorted());
std::sort(v.begin(), v.end(), cmp);
} else {
v = ConsumeRawPtr(top.Extract());
}
EXPECT_EQ(top_size, v.size());
for (int i = 0; i != top_size; ++i) {
VLOG(1) << "Top element " << v[i];
EXPECT_EQ(shadow[i], v[i]);
}
}
template <class Cmp>
void TestIntTopN(size_t limit, size_t n_elements, const Cmp &cmp,
SimplePhilox *random) {
// Test peek_bottom() and Extract()
TestIntTopNHelper(limit, n_elements, cmp, random, true, false);
// Test Extract()
TestIntTopNHelper(limit, n_elements, cmp, random, false, false);
// Test peek_bottom() and ExtractUnsorted()
TestIntTopNHelper(limit, n_elements, cmp, random, true, true);
// Test ExtractUnsorted()
TestIntTopNHelper(limit, n_elements, cmp, random, false, true);
}
TEST(TopNTest, Misc) {
PhiloxRandom philox(1, 1);
SimplePhilox random(&philox);
TestIntTopN(0, 5, std::greater<int>(), &random);
TestIntTopN(32, 0, std::greater<int>(), &random);
TestIntTopN(6, 6, std::greater<int>(), &random);
TestIntTopN(6, 6, std::less<int>(), &random);
TestIntTopN(1000, 999, std::greater<int>(), &random);
TestIntTopN(1000, 1000, std::greater<int>(), &random);
TestIntTopN(1000, 1001, std::greater<int>(), &random);
TestIntTopN(2300, 28393, std::less<int>(), &random);
TestIntTopN(30, 100, std::greater<int>(), &random);
TestIntTopN(100, 30, std::less<int>(), &random);
TestIntTopN(size_t(-1), 3, std::greater<int>(), &random);
TestIntTopN(size_t(-1), 0, std::greater<int>(), &random);
TestIntTopN(0, 5, std::greater<int>(), &random);
}
TEST(TopNTest, String) {
LOG(INFO) << "Testing strings";
TopN<string> top(3);
EXPECT_TRUE(top.empty());
top.push("abracadabra");
top.push("waldemar");
EXPECT_EQ(2, top.size());
EXPECT_EQ("abracadabra", top.peek_bottom());
top.push("");
EXPECT_EQ(3, top.size());
EXPECT_EQ("", top.peek_bottom());
top.push("top");
EXPECT_EQ(3, top.size());
EXPECT_EQ("abracadabra", top.peek_bottom());
top.push("Google");
top.push("test");
EXPECT_EQ(3, top.size());
EXPECT_EQ("test", top.peek_bottom());
TopN<string> top2(top);
TopN<string> top3(5);
top3 = top;
EXPECT_EQ("test", top3.peek_bottom());
{
std::vector<string> s = ConsumeRawPtr(top.Extract());
EXPECT_EQ(s[0], "waldemar");
EXPECT_EQ(s[1], "top");
EXPECT_EQ(s[2], "test");
}
top2.push("zero");
EXPECT_EQ(top2.peek_bottom(), "top");
{
std::vector<string> s = ConsumeRawPtr(top2.Extract());
EXPECT_EQ(s[0], "zero");
EXPECT_EQ(s[1], "waldemar");
EXPECT_EQ(s[2], "top");
}
{
std::vector<string> s = ConsumeRawPtr(top3.Extract());
EXPECT_EQ(s[0], "waldemar");
EXPECT_EQ(s[1], "top");
EXPECT_EQ(s[2], "test");
}
TopN<string> top4(3);
// Run this test twice to check Reset():
for (int i = 0; i < 2; ++i) {
top4.push("abcd");
top4.push("ijkl");
top4.push("efgh");
top4.push("mnop");
std::vector<string> s = ConsumeRawPtr(top4.Extract());
EXPECT_EQ(s[0], "mnop");
EXPECT_EQ(s[1], "ijkl");
EXPECT_EQ(s[2], "efgh");
top4.Reset();
}
}
// Test that pointers aren't leaked from a TopN if we use the 2-argument version
// of push().
TEST(TopNTest, Ptr) {
LOG(INFO) << "Testing 2-argument push()";
TopN<string *> topn(3);
for (int i = 0; i < 8; ++i) {
string *dropped = nullptr;
topn.push(new string(std::to_string(i)), &dropped);
delete dropped;
}
for (int i = 8; i > 0; --i) {
string *dropped = nullptr;
topn.push(new string(std::to_string(i)), &dropped);
delete dropped;
}
std::vector<string *> extract = ConsumeRawPtr(topn.Extract());
for (auto &temp : extract) {
delete temp;
}
extract.clear();
}
struct PointeeGreater {
template <typename T>
bool operator()(const T &a, const T &b) const {
return *a > *b;
}
};
TEST(TopNTest, MoveOnly) {
using StrPtr = std::unique_ptr<string>;
TopN<StrPtr, PointeeGreater> topn(3);
for (int i = 0; i < 8; ++i) topn.push(StrPtr(new string(std::to_string(i))));
for (int i = 8; i > 0; --i) topn.push(StrPtr(new string(std::to_string(i))));
std::vector<StrPtr> extract = ConsumeRawPtr(topn.Extract());
EXPECT_EQ(extract.size(), 3);
EXPECT_EQ(*(extract[0]), "8");
EXPECT_EQ(*(extract[1]), "7");
EXPECT_EQ(*(extract[2]), "7");
}
// Test that Nondestructive extracts do not need a Reset() afterwards,
// and that pointers aren't leaked from a TopN after calling them.
TEST(TopNTest, Nondestructive) {
LOG(INFO) << "Testing Nondestructive extracts";
TopN<int> top4(4);
for (int i = 0; i < 8; ++i) {
top4.push(i);
std::vector<int> v = ConsumeRawPtr(top4.ExtractNondestructive());
EXPECT_EQ(std::min(i + 1, 4), v.size());
for (size_t j = 0; j < v.size(); ++j) EXPECT_EQ(i - j, v[j]);
}
TopN<int> top3(3);
for (int i = 0; i < 8; ++i) {
top3.push(i);
std::vector<int> v = ConsumeRawPtr(top3.ExtractUnsortedNondestructive());
std::sort(v.begin(), v.end(), std::greater<int>());
EXPECT_EQ(std::min(i + 1, 3), v.size());
for (size_t j = 0; j < v.size(); ++j) EXPECT_EQ(i - j, v[j]);
}
}
struct ForbiddenCmp {
bool operator()(int lhs, int rhs) const {
LOG(FATAL) << "ForbiddenCmp called " << lhs << " " << rhs;
}
};
TEST(TopNTest, ZeroLimit) {
TopN<int, ForbiddenCmp> top(0);
top.push(1);
top.push(2);
int dropped = -1;
top.push(1, &dropped);
top.push(2, &dropped);
std::vector<int> v;
top.ExtractNondestructive(&v);
EXPECT_EQ(0, v.size());
}
TEST(TopNTest, Iteration) {
TopN<int> top(4);
for (int i = 0; i < 8; ++i) top.push(i);
std::vector<int> actual(top.unsorted_begin(), top.unsorted_end());
// Check that we have 4,5,6,7 as the top 4 (in some order, so we sort)
std::sort(actual.begin(), actual.end());
EXPECT_EQ(actual.size(), 4);
EXPECT_EQ(actual[0], 4);
EXPECT_EQ(actual[1], 5);
EXPECT_EQ(actual[2], 6);
EXPECT_EQ(actual[3], 7);
}
} // namespace
tensorflow/core/lib/hash/BUILD
+83
View File
@@ -0,0 +1,83 @@
load("//tensorflow:tensorflow.default.bzl", "filegroup")
load(
"//tensorflow/core/platform:rules_cc.bzl",
"cc_library",
)
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = [
# tensorflow/core:lib effectively exposes all targets under tensorflow/core/lib/**
"//tensorflow/core:__pkg__",
# tensorflow/core/lib/gtl uses hash functions as part of their flatset/map representation
"//tensorflow/core/lib/gtl:__pkg__",
# tensorflow/core/lib/io/table_builder.cc uses crc functionality
"//tensorflow/core/lib/io:__pkg__",
# tensorflow/core/framework:tensor depends on hash
"//tensorflow/core/framework:__pkg__",
],
licenses = ["notice"],
)
cc_library(
name = "crc32c",
hdrs = ["crc32c.h"],
deps = [
"//tensorflow/core/platform",
"//tensorflow/core/platform:cord",
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/hash:crc32c",
],
)
cc_library(
name = "hash",
hdrs = ["hash.h"],
deps = [
"//tensorflow/core/platform:hash",
],
)
# Export source files needed for mobile builds, which do not use granular targets.
filegroup(
name = "mobile_hdrs_no_runtime",
srcs = [
"hash.h",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "mobile_srcs_only_runtime",
srcs = [
"@xla//xla/tsl/lib/hash:mobile_srcs_only_runtime",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "mobile_hdrs_only_runtime",
srcs = [
"crc32c.h",
"@xla//xla/tsl/lib/hash:mobile_hdrs_only_runtime",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_hash_all_headers",
srcs = [
"crc32c.h",
"hash.h",
"@xla//xla/tsl/lib/hash:legacy_lib_hash_all_headers",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_internal_public_headers",
srcs = [
"hash.h",
],
visibility = ["//tensorflow/core:__pkg__"],
)
tensorflow/core/lib/hash/crc32c.h
+38
View File
@@ -0,0 +1,38 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_HASH_CRC32C_H_
#define TENSORFLOW_CORE_LIB_HASH_CRC32C_H_
#include <stddef.h>
#include "xla/tsl/lib/hash/crc32c.h"
#include "tensorflow/core/platform/cord.h"
#include "tensorflow/core/platform/platform.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace crc32c {
// NOLINTBEGIN(misc-unused-using-decls)
using tsl::crc32c::Extend;
using tsl::crc32c::kMaskDelta;
using tsl::crc32c::Mask;
using tsl::crc32c::Unmask;
using tsl::crc32c::Value;
// NOLINTEND(misc-unused-using-decls)
} // namespace crc32c
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_HASH_CRC32C_H_
tensorflow/core/lib/hash/hash.h
+23
View File
@@ -0,0 +1,23 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Simple hash functions used for internal data structures
#ifndef TENSORFLOW_CORE_LIB_HASH_HASH_H_
#define TENSORFLOW_CORE_LIB_HASH_HASH_H_
#include "tensorflow/core/platform/hash.h"
#endif // TENSORFLOW_CORE_LIB_HASH_HASH_H_
tensorflow/core/lib/histogram/BUILD
+58
View File
@@ -0,0 +1,58 @@
load("//tensorflow:tensorflow.default.bzl", "filegroup")
load(
"//tensorflow/core/platform:rules_cc.bzl",
"cc_library",
)
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = [
# tensorflow/core:lib effectively exposes all targets under tensorflow/core/lib/**
"//tensorflow/core:__pkg__",
# tensorflow/core/lib/monitoring:sampler uses histogram
"//tensorflow/core/lib/monitoring:__pkg__",
],
licenses = ["notice"],
)
cc_library(
name = "histogram",
hdrs = ["histogram.h"],
deps = [
"//tensorflow/core/lib/gtl:array_slice",
"//tensorflow/core/platform:logging",
"//tensorflow/core/platform:macros",
"//tensorflow/core/platform:mutex",
"//tensorflow/core/platform:thread_annotations",
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/histogram",
],
alwayslink = True,
)
# Export source files needed for mobile builds, which do not use granular targets.
filegroup(
name = "mobile_srcs_only_runtime",
srcs = [
"@xla//xla/tsl/lib/histogram:mobile_srcs_only_runtime",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "mobile_hdrs_only_runtime",
srcs = [
"histogram.h",
"@xla//xla/tsl/lib/histogram:mobile_hdrs_only_runtime",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_histogram_all_headers",
srcs = [
"histogram.h",
"@xla//xla/tsl/lib/histogram:legacy_lib_histogram_all_headers",
],
visibility = ["//tensorflow/core:__pkg__"],
)
tensorflow/core/lib/histogram/histogram.h
+41
View File
@@ -0,0 +1,41 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_HISTOGRAM_HISTOGRAM_H_
#define TENSORFLOW_CORE_LIB_HISTOGRAM_HISTOGRAM_H_
#include <string>
#include <vector>
#include "xla/tsl/lib/histogram/histogram.h"
#include "tensorflow/core/lib/gtl/array_slice.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
using tsl::HistogramProto; // NOLINT
namespace histogram {
using tsl::histogram::Histogram; // NOLINT
using tsl::histogram::ThreadSafeHistogram; // NOLINT
} // namespace histogram
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_HISTOGRAM_HISTOGRAM_H_
tensorflow/core/lib/io/BUILD
+325
View File
@@ -0,0 +1,325 @@
load("//tensorflow:tensorflow.default.bzl", "filegroup")
load(
"//tensorflow/core/platform:rules_cc.bzl",
"cc_library",
)
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
default_visibility = [
"//tensorflow/c/experimental/filesystem:__pkg__",
"//third_party/py/tensorflow_io:__subpackages__",
# tensorflow/core:lib effectively exposes all targets under tensorflow/core/lib/**
"//tensorflow/core:__pkg__",
],
licenses = ["notice"],
)
cc_library(
name = "block",
hdrs = [
"block.h",
"block_builder.h",
"format.h",
"table_builder.h",
],
deps = [
":iterator",
":table_options",
"//tensorflow/core/platform:status",
"//tensorflow/core/platform:stringpiece",
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/io:block",
],
)
cc_library(
name = "buffered_inputstream",
hdrs = ["buffered_inputstream.h"],
deps = [
":inputstream_interface",
"//tensorflow/core/platform:env",
"@xla//xla/tsl/lib/io:buffered_inputstream",
],
)
cc_library(
name = "compression",
hdrs = ["compression.h"],
deps = ["@xla//xla/tsl/lib/io:compression"],
)
cc_library(
name = "inputbuffer",
hdrs = ["inputbuffer.h"],
deps = [
"//tensorflow/core/platform:coding",
"//tensorflow/core/platform:env",
"//tensorflow/core/platform:macros",
"//tensorflow/core/platform:status",
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/io:inputbuffer",
],
)
cc_library(
name = "inputstream_interface",
hdrs = ["inputstream_interface.h"],
deps = [
"//tensorflow/core/platform:cord",
"//tensorflow/core/platform:errors",
"//tensorflow/core/platform:status",
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/io:inputstream_interface",
],
)
cc_library(
name = "iterator",
hdrs = ["iterator.h"],
deps = [
"//tensorflow/core/platform:status",
"//tensorflow/core/platform:stringpiece",
"@xla//xla/tsl/lib/io:iterator",
],
)
cc_library(
name = "path",
hdrs = ["path.h"],
deps = ["//tensorflow/core/platform:path"],
)
cc_library(
name = "proto_encode_helper",
hdrs = ["proto_encode_helper.h"],
deps = [
"//tensorflow/core/platform:coding",
"//tensorflow/core/platform:logging",
"//tensorflow/core/platform:protobuf",
"//tensorflow/core/platform:stringpiece",
"@xla//xla/tsl/lib/io:proto_encode_helper",
],
)
cc_library(
name = "random_inputstream",
hdrs = ["random_inputstream.h"],
deps = [
":inputstream_interface",
"//tensorflow/core/platform:cord",
"//tensorflow/core/platform:env",
"@xla//xla/tsl/lib/io:random_inputstream",
],
)
cc_library(
name = "record_reader",
hdrs = ["record_reader.h"],
deps = [
":inputstream_interface",
":zlib_compression_options",
":zlib_inputstream",
"//tensorflow/core/platform:env",
"//tensorflow/core/platform:errors",
"//tensorflow/core/platform:macros",
"//tensorflow/core/platform:stringpiece",
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/io:record_reader",
],
)
cc_library(
name = "record_writer",
hdrs = ["record_writer.h"],
deps = [
":compression",
":zlib_compression_options",
":zlib_outputbuffer",
"//tensorflow/core/lib/hash:crc32c",
"//tensorflow/core/platform:coding",
"//tensorflow/core/platform:cord",
"//tensorflow/core/platform:macros",
"//tensorflow/core/platform:status",
"//tensorflow/core/platform:stringpiece",
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/io:record_writer",
],
)
alias(
name = "snappy_inputbuffer",
actual = "@xla//xla/tsl/lib/io/snappy:snappy_inputbuffer",
)
alias(
name = "snappy_inputstream",
actual = "@xla//xla/tsl/lib/io/snappy:snappy_inputstream",
)
alias(
name = "snappy_outputbuffer",
actual = "@xla//xla/tsl/lib/io/snappy:snappy_outputbuffer",
)
alias(
name = "snappy_compression_options",
actual = "@xla//xla/tsl/lib/io/snappy:snappy_compression_options",
)
cc_library(
name = "cache",
hdrs = ["cache.h"],
deps = [
"//tensorflow/core/platform:stringpiece",
"@xla//xla/tsl/lib/io:cache",
],
)
cc_library(
name = "table",
hdrs = [
"table.h",
"two_level_iterator.h",
],
deps = [
":iterator",
"@xla//xla/tsl/lib/io:table",
],
)
cc_library(
name = "table_options",
hdrs = ["table_options.h"],
deps = ["@xla//xla/tsl/lib/io:table_options"],
)
cc_library(
name = "zlib_compression_options",
hdrs = ["zlib_compression_options.h"],
deps = [
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/io:zlib_compression_options",
],
)
cc_library(
name = "zlib_inputstream",
hdrs = ["zlib_inputstream.h"],
deps = [
":inputstream_interface",
":zlib_compression_options",
"//tensorflow/core/platform:env",
"//tensorflow/core/platform:macros",
"//tensorflow/core/platform:status",
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/io:zlib_inputstream",
],
)
cc_library(
name = "zlib_outputbuffer",
hdrs = ["zlib_outputbuffer.h"],
deps = [
":zlib_compression_options",
"//tensorflow/core/platform:env",
"//tensorflow/core/platform:macros",
"//tensorflow/core/platform:status",
"//tensorflow/core/platform:stringpiece",
"//tensorflow/core/platform:types",
"@xla//xla/tsl/lib/io:zlib_outputbuffer",
],
)
# Export source files needed for mobile builds, which do not use granular targets.
filegroup(
name = "mobile_hdrs_only_runtime",
srcs = [
"block.h",
"block_builder.h",
"buffered_inputstream.h",
"cache.h",
"compression.h",
"format.h",
"inputbuffer.h",
"inputstream_interface.h",
"iterator.h",
"path.h",
"random_inputstream.h",
"record_reader.h",
"table.h",
"table_builder.h",
"table_options.h",
"two_level_iterator.h",
"zlib_compression_options.h",
"zlib_inputstream.h",
],
)
filegroup(
name = "legacy_lib_io_all_headers",
srcs = [
"block.h",
"block_builder.h",
"buffered_inputstream.h",
"compression.h",
"format.h",
"inputbuffer.h",
"inputstream_interface.h",
"iterator.h",
"path.h",
"proto_encode_helper.h",
"random_inputstream.h",
"record_reader.h",
"record_writer.h",
"table.h",
"table_builder.h",
"table_options.h",
"two_level_iterator.h",
"zlib_compression_options.h",
"zlib_inputstream.h",
"zlib_outputbuffer.h",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_io_headers",
srcs = [
"buffered_inputstream.h",
"cache.h",
"compression.h",
"inputstream_interface.h",
"path.h",
"proto_encode_helper.h",
"random_inputstream.h",
"record_reader.h",
"record_writer.h",
"table.h",
"table_builder.h",
"table_options.h",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_internal_public_headers",
srcs = [
"inputbuffer.h",
"iterator.h",
"zlib_compression_options.h",
"zlib_inputstream.h",
"zlib_outputbuffer.h",
],
visibility = ["//tensorflow/core:__pkg__"],
)
filegroup(
name = "legacy_lib_test_internal_headers",
srcs = [
"block.h",
"block_builder.h",
"format.h",
],
visibility = ["//tensorflow/core:__pkg__"],
)
tensorflow/core/lib/io/block.h
+28
View File
@@ -0,0 +1,28 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_BLOCK_H_
#define TENSORFLOW_CORE_LIB_IO_BLOCK_H_
#include "xla/tsl/lib/io/block.h"
#include "tensorflow/core/lib/io/iterator.h"
namespace tensorflow {
namespace table {
using tsl::table::Block; // NOLINT(misc-unused-using-decls)
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_BLOCK_H_
tensorflow/core/lib/io/block_builder.h
+29
View File
@@ -0,0 +1,29 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_BLOCK_BUILDER_H_
#define TENSORFLOW_CORE_LIB_IO_BLOCK_BUILDER_H_
#include "xla/tsl/lib/io/block_builder.h"
#include "tensorflow/core/platform/stringpiece.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace table {
using tsl::table::BlockBuilder; // NOLINT(misc-unused-using-decls)
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_BLOCK_BUILDER_H_
tensorflow/core/lib/io/buffered_inputstream.h
+29
View File
@@ -0,0 +1,29 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_BUFFERED_INPUTSTREAM_H_
#define TENSORFLOW_CORE_LIB_IO_BUFFERED_INPUTSTREAM_H_
#include "xla/tsl/lib/io/buffered_inputstream.h"
#include "tensorflow/core/lib/io/inputstream_interface.h"
#include "tensorflow/core/platform/file_system.h"
namespace tensorflow {
namespace io {
using tsl::io::BufferedInputStream; // NOLINT(misc-unused-using-decls)
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_BUFFERED_INPUTSTREAM_H_
tensorflow/core/lib/io/cache.h
+32
View File
@@ -0,0 +1,32 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_CACHE_H_
#define TENSORFLOW_CORE_LIB_IO_CACHE_H_
#include "xla/tsl/lib/io/cache.h"
#include "tensorflow/core/platform/stringpiece.h"
namespace tensorflow {
using tsl::Slice; // NOLINT(misc-unused-using-decls)
namespace table {
// NOLINTBEGIN(misc-unused-using-decls)
using tsl::table::Cache;
using tsl::table::NewLRUCache;
// NOLINTEND(misc-unused-using-decls)
} // namespace table
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_CACHE_H_
tensorflow/core/lib/io/compression.h
+34
View File
@@ -0,0 +1,34 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_COMPRESSION_H_
#define TENSORFLOW_CORE_LIB_IO_COMPRESSION_H_
#include "xla/tsl/lib/io/compression.h"
namespace tensorflow {
namespace io {
namespace compression {
// NOLINTBEGIN(misc-unused-using-decls)
using tsl::io::compression::kGzip;
using tsl::io::compression::kNone;
using tsl::io::compression::kSnappy;
using tsl::io::compression::kZlib;
// NOLINTEND(misc-unused-using-decls)
} // namespace compression
} // namespace io
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_COMPRESSION_H_
tensorflow/core/lib/io/format.h
+36
View File
@@ -0,0 +1,36 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_FORMAT_H_
#define TENSORFLOW_CORE_LIB_IO_FORMAT_H_
#include "xla/tsl/lib/io/format.h"
#include "tensorflow/core/lib/io/table_builder.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/stringpiece.h"
namespace tensorflow {
namespace table {
// NOLINTBEGIN(misc-unused-using-decls)
using tsl::table::BlockContents;
using tsl::table::BlockHandle;
using tsl::table::kBlockTrailerSize;
using tsl::table::kTableMagicNumber;
using tsl::table::ReadBlock;
// NOLINTEND(misc-unused-using-decls)
} // namespace table
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_FORMAT_H_
tensorflow/core/lib/io/inputbuffer.h
+32
View File
@@ -0,0 +1,32 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_INPUTBUFFER_H_
#define TENSORFLOW_CORE_LIB_IO_INPUTBUFFER_H_
#include "xla/tsl/lib/io/inputbuffer.h"
#include "tensorflow/core/platform/coding.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace io {
using tsl::io::InputBuffer; // NOLINT(misc-unused-using-decls)
}
}
#endif // TENSORFLOW_CORE_LIB_IO_INPUTBUFFER_H_
tensorflow/core/lib/io/inputstream_interface.h
+31
View File
@@ -0,0 +1,31 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_INPUTSTREAM_INTERFACE_H_
#define TENSORFLOW_CORE_LIB_IO_INPUTSTREAM_INTERFACE_H_
#include "xla/tsl/lib/io/inputstream_interface.h"
#include "tensorflow/core/platform/cord.h"
#include "tensorflow/core/platform/errors.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace io {
using tsl::io::InputStreamInterface; // NOLINT(misc-unused-using-decls)
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_INPUTSTREAM_INTERFACE_H_
tensorflow/core/lib/io/iterator.h
+43
View File
@@ -0,0 +1,43 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// An iterator yields a sequence of key/value pairs from a source.
// The following class defines the interface. Multiple implementations
// are provided by this library. In particular, iterators are provided
// to access the contents of a Table or a DB.
//
// Multiple threads can invoke const methods on an Iterator without
// external synchronization, but if any of the threads may call a
// non-const method, all threads accessing the same Iterator must use
// external synchronization.
#ifndef TENSORFLOW_CORE_LIB_IO_ITERATOR_H_
#define TENSORFLOW_CORE_LIB_IO_ITERATOR_H_
#include "xla/tsl/lib/io/iterator.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/stringpiece.h"
namespace tensorflow {
namespace table {
// NOLINTBEGIN(misc-unused-using-decls)
using tsl::table::Iterator;
using tsl::table::NewEmptyIterator;
using tsl::table::NewErrorIterator;
// NOLINTEND(misc-unused-using-decls)
} // namespace table
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_ITERATOR_H_
tensorflow/core/lib/io/path.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_PATH_H_
#define TENSORFLOW_CORE_LIB_IO_PATH_H_
#include "tensorflow/core/platform/path.h" // IWYU pragma: export
#endif // TENSORFLOW_CORE_LIB_IO_PATH_H_
tensorflow/core/lib/io/proto_encode_helper.h
+31
View File
@@ -0,0 +1,31 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_PROTO_ENCODE_HELPER_H_
#define TENSORFLOW_CORE_LIB_IO_PROTO_ENCODE_HELPER_H_
#include "xla/tsl/lib/io/proto_encode_helper.h"
#include "tensorflow/core/platform/coding.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/stringpiece.h"
namespace tensorflow {
namespace io {
using tsl::io::ProtoEncodeHelper; // NOLINT(misc-unused-using-decls)
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_PROTO_ENCODE_HELPER_H_
tensorflow/core/lib/io/random_inputstream.h
+30
View File
@@ -0,0 +1,30 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_RANDOM_INPUTSTREAM_H_
#define TENSORFLOW_CORE_LIB_IO_RANDOM_INPUTSTREAM_H_
#include "xla/tsl/lib/io/random_inputstream.h"
#include "tensorflow/core/lib/io/inputstream_interface.h"
#include "tensorflow/core/platform/cord.h"
#include "tensorflow/core/platform/file_system.h"
namespace tensorflow {
namespace io {
using tsl::io::RandomAccessInputStream; // NOLINT(misc-unused-using-decls)
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_RANDOM_INPUTSTREAM_H_
tensorflow/core/lib/io/record_reader.h
+40
View File
@@ -0,0 +1,40 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_RECORD_READER_H_
#define TENSORFLOW_CORE_LIB_IO_RECORD_READER_H_
#include "tensorflow/core/lib/io/inputstream_interface.h"
#include "tensorflow/core/platform/errors.h"
#include "tensorflow/core/platform/stringpiece.h"
#if !defined(IS_SLIM_BUILD)
#include "tensorflow/core/lib/io/zlib_compression_options.h"
#include "tensorflow/core/lib/io/zlib_inputstream.h"
#endif // IS_SLIM_BUILD
#include "xla/tsl/lib/io/record_reader.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace io {
// NOLINTBEGIN(misc-unused-using-decls)
using tsl::io::RecordReader;
using tsl::io::RecordReaderOptions;
using tsl::io::SequentialRecordReader;
// NOLINTEND(misc-unused-using-decls)
} // namespace io
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_RECORD_READER_H_
tensorflow/core/lib/io/record_writer.h
+41
View File
@@ -0,0 +1,41 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_RECORD_WRITER_H_
#define TENSORFLOW_CORE_LIB_IO_RECORD_WRITER_H_
#include "tensorflow/core/lib/hash/crc32c.h"
#include "tensorflow/core/platform/coding.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/stringpiece.h"
#if !defined(IS_SLIM_BUILD)
#include "tensorflow/core/lib/io/zlib_compression_options.h"
#include "tensorflow/core/lib/io/zlib_outputbuffer.h"
#endif // IS_SLIM_BUILD
#include "xla/tsl/lib/io/record_writer.h"
#include "tensorflow/core/platform/cord.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace io {
// NOLINTBEGIN(misc-unused-using-decls)
using tsl::io::RecordWriter;
using tsl::io::RecordWriterOptions;
// NOLINTEND(misc-unused-using-decls)
} // namespace io
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_RECORD_WRITER_H_
tensorflow/core/lib/io/table.h
+28
View File
@@ -0,0 +1,28 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_TABLE_H_
#define TENSORFLOW_CORE_LIB_IO_TABLE_H_
#include "xla/tsl/lib/io/table.h"
#include "tensorflow/core/lib/io/iterator.h"
namespace tensorflow {
namespace table {
using tsl::table::Table; // NOLINT(misc-unused-using-decls)
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_TABLE_H_
tensorflow/core/lib/io/table_builder.h
+38
View File
@@ -0,0 +1,38 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// TableBuilder provides the interface used to build a Table
// (an immutable and sorted map from keys to values).
//
// Multiple threads can invoke const methods on a TableBuilder without
// external synchronization, but if any of the threads may call a
// non-const method, all threads accessing the same TableBuilder must use
// external synchronization.
#ifndef TENSORFLOW_CORE_LIB_IO_TABLE_BUILDER_H_
#define TENSORFLOW_CORE_LIB_IO_TABLE_BUILDER_H_
#include "xla/tsl/lib/io/table_builder.h"
#include "tensorflow/core/lib/io/table_options.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/stringpiece.h"
namespace tensorflow {
namespace table {
using tsl::table::TableBuilder; // NOLINT(misc-unused-using-decls)
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_TABLE_BUILDER_H_
tensorflow/core/lib/io/table_options.h
+32
View File
@@ -0,0 +1,32 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_TABLE_OPTIONS_H_
#define TENSORFLOW_CORE_LIB_IO_TABLE_OPTIONS_H_
#include "xla/tsl/lib/io/table_options.h"
namespace tensorflow {
namespace table {
// NOLINTBEGIN(misc-unused-using-decls)
using tsl::table::CompressionType;
using tsl::table::kNoCompression;
using tsl::table::kSnappyCompression;
using tsl::table::Options;
// NOLINTEND(misc-unused-using-decls)
} // namespace table
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_TABLE_OPTIONS_H_
tensorflow/core/lib/io/two_level_iterator.h
+28
View File
@@ -0,0 +1,28 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_TWO_LEVEL_ITERATOR_H_
#define TENSORFLOW_CORE_LIB_IO_TWO_LEVEL_ITERATOR_H_
#include "xla/tsl/lib/io/two_level_iterator.h"
#include "tensorflow/core/lib/io/iterator.h"
namespace tensorflow {
namespace table {
using tsl::table::NewTwoLevelIterator; // NOLINT(misc-unused-using-decls)
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_TWO_LEVEL_ITERATOR_H_
tensorflow/core/lib/io/zlib_compression_options.h
+28
View File
@@ -0,0 +1,28 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_ZLIB_COMPRESSION_OPTIONS_H_
#define TENSORFLOW_CORE_LIB_IO_ZLIB_COMPRESSION_OPTIONS_H_
#include "xla/tsl/lib/io/zlib_compression_options.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace io {
using tsl::io::ZlibCompressionOptions; // NOLINT(misc-unused-using-decls)
}
}
#endif // TENSORFLOW_CORE_LIB_IO_ZLIB_COMPRESSION_OPTIONS_H_
tensorflow/core/lib/io/zlib_inputstream.h
+33
View File
@@ -0,0 +1,33 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_LIB_IO_ZLIB_INPUTSTREAM_H_
#define TENSORFLOW_CORE_LIB_IO_ZLIB_INPUTSTREAM_H_
#include "xla/tsl/lib/io/zlib_inputstream.h"
#include "tensorflow/core/lib/io/inputstream_interface.h"
#include "tensorflow/core/lib/io/zlib_compression_options.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace io {
using tsl::io::ZlibInputStream; // NOLINT(misc-unused-using-decls);
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_LIB_IO_ZLIB_INPUTSTREAM_H_

Some files were not shown because too many files have changed in this diff Show More