diff --git a/tensorflow_text/core/kernels/constrained_sequence.cc b/tensorflow_text/core/kernels/constrained_sequence.cc index 2553472c1..74037c023 100644 --- a/tensorflow_text/core/kernels/constrained_sequence.cc +++ b/tensorflow_text/core/kernels/constrained_sequence.cc @@ -38,7 +38,7 @@ ScoreAccessor::ScoreAccessor(const Tensor &score_tensor, data_ = score_tensor.flat().data(); if (lengths_tensor.dtype() == DT_INT64) { use_long_lengths_ = true; - long_lengths_ = lengths_tensor.flat().data(); + long_lengths_ = lengths_tensor.flat().data(); } else { use_long_lengths_ = false; lengths_ = lengths_tensor.flat().data(); @@ -66,7 +66,7 @@ float ScoreAccessor::GetScore(int batch_idx, int step_idx, return data_[batch_offset_ * batch_idx + step_offset_ * step_idx + score_idx]; } -int64 ScoreAccessor::GetLength(int batch_idx) const { +int64_t ScoreAccessor::GetLength(int batch_idx) const { DCHECK_LE(batch_idx, batch_size_); if (use_long_lengths_) { return long_lengths_[batch_idx]; @@ -82,18 +82,18 @@ bool ScoreAccessor::has_explicit_batch() const { return has_explicit_batch_; } // Perform Viterbi analysis on a single batch item. void ViterbiAnalysis( - const ScoreAccessor &scores, - const tensorflow::TTypes::Matrix &transition_weights, - const tensorflow::TTypes::Matrix &allowed_transitions, + const ScoreAccessor& scores, + const tensorflow::TTypes::Matrix& transition_weights, + const tensorflow::TTypes::Matrix& allowed_transitions, const int batch, bool use_log_space, bool use_start_end_states, - int32 *output_data) { + int32_t* output_data) { VLOG(2) << "Analyzing batch " << batch; const bool has_transition_weights = transition_weights.size() != 0; const bool has_allowed_transitions = allowed_transitions.size() != 0; const int num_states = scores.num_scores(); const int out_of_bounds_index = num_states; - int64 num_steps = scores.GetLength(batch); + int64_t num_steps = scores.GetLength(batch); // Create two vectors to hold scores. These will be bound to referents later // so the names here are somewhat irrelevant. @@ -344,14 +344,14 @@ void ViterbiAnalysis( if (best_source_state == kErrorState) { // If the best source is an error state, the path is unknowable. Report // error states for the whole sequence. - for (int64 i = 0; i < scores.GetLength(batch); ++i) { + for (int64_t i = 0; i < scores.GetLength(batch); ++i) { output_data[i] = kErrorState; } } else { // If the best source is a 'real' state, report the state path. int steps_to_report = scores.GetLength(batch); int previous_state = best_source_state; - for (int64 i = steps_to_report - 1; i >= 0; --i) { + for (int64_t i = steps_to_report - 1; i >= 0; --i) { output_data[i] = previous_state; previous_state = backpointers[i][previous_state]; } @@ -359,16 +359,16 @@ void ViterbiAnalysis( } void GreedyAnalysis( - const ScoreAccessor &scores, - const tensorflow::TTypes::Matrix &transition_weights, - const tensorflow::TTypes::Matrix &allowed_transitions, + const ScoreAccessor& scores, + const tensorflow::TTypes::Matrix& transition_weights, + const tensorflow::TTypes::Matrix& allowed_transitions, int batch, bool use_log_space, bool use_start_end_states, - int32 *output_data) { + int32_t* output_data) { const bool has_transition_weights = transition_weights.size() != 0; const bool has_allowed_transitions = allowed_transitions.size() != 0; const int num_states = scores.num_scores(); const int out_of_bounds_index = num_states; - int64 num_steps = scores.GetLength(batch); + int64_t num_steps = scores.GetLength(batch); for (int step = 0; step < num_steps; ++step) { // Do final step calculations if this is the final step in the sequence diff --git a/tensorflow_text/core/kernels/constrained_sequence.h b/tensorflow_text/core/kernels/constrained_sequence.h index 5f62f46b6..09baeb6ee 100644 --- a/tensorflow_text/core/kernels/constrained_sequence.h +++ b/tensorflow_text/core/kernels/constrained_sequence.h @@ -30,7 +30,7 @@ class ScoreAccessor { // Get a score out of the data tensor. float GetScore(int batch_idx, int step_idx, int score_idx) const; - int64 GetLength(int batch_idx) const; + int64_t GetLength(int batch_idx) const; int batch_size() const; int num_steps() const; @@ -43,7 +43,7 @@ class ScoreAccessor { // A pointer into the underlying data of the lengths tensor. Not owned. const int *lengths_; - const int64 *long_lengths_; + const int64_t* long_lengths_; // Whether the passed lengths tensor is int32 or int64. bool use_long_lengths_; @@ -72,19 +72,19 @@ class ScoreAccessor { // Perform Viterbi analysis on a single batch item. void ViterbiAnalysis( - const ScoreAccessor &scores, - const tensorflow::TTypes::Matrix &transition_weights, - const tensorflow::TTypes::Matrix &allowed_transitions, + const ScoreAccessor& scores, + const tensorflow::TTypes::Matrix& transition_weights, + const tensorflow::TTypes::Matrix& allowed_transitions, const int batch, bool use_log_space, bool use_start_end_states, - int32 *output_data); + int32_t* output_data); // Perform a greedy analysis on a single batch item. void GreedyAnalysis( - const ScoreAccessor &scores, - const tensorflow::TTypes::Matrix &transition_weights, - const tensorflow::TTypes::Matrix &allowed_transitions, + const ScoreAccessor& scores, + const tensorflow::TTypes::Matrix& transition_weights, + const tensorflow::TTypes::Matrix& allowed_transitions, int batch, bool use_log_space, bool use_start_end_states, - int32 *output_data); + int32_t* output_data); } // namespace text } // namespace tensorflow diff --git a/tensorflow_text/core/kernels/constrained_sequence_kernel.cc b/tensorflow_text/core/kernels/constrained_sequence_kernel.cc index 339c202ce..0d0359275 100644 --- a/tensorflow_text/core/kernels/constrained_sequence_kernel.cc +++ b/tensorflow_text/core/kernels/constrained_sequence_kernel.cc @@ -57,13 +57,12 @@ namespace { // Validate that a given constraint tensor is the proper shape (dimension // 2, with shape [num_states + 1, num_states + 1]. -absl::Status ValidateConstraintTensor(const Tensor &tensor, +absl::Status ValidateConstraintTensor(const Tensor& tensor, const int num_states, const bool use_start_end_states, - const string &name) { + const std::string& name) { if (tensor.shape().dims() != 2) { - return InvalidArgument( - tensorflow::strings::StrCat(name, " must be of rank 2")); + return InvalidArgument(absl::StrCat(name, " must be of rank 2")); } int expected_size = use_start_end_states ? num_states + 1 : num_states; if (tensor.shape().dim_size(0) != expected_size) { @@ -110,7 +109,7 @@ class ConstrainedSequence : public OpKernel { const int num_scores = scores.num_scores(); OP_REQUIRES(context, lengths_tensor.NumElements() == batch_size, - InvalidArgument(tensorflow::strings::StrCat( + InvalidArgument(absl::StrCat( "There should be exactly one length for every batch " "element. Found ", lengths_tensor.NumElements(), @@ -124,7 +123,7 @@ class ConstrainedSequence : public OpKernel { int max_length = 0; int total_length = 0; for (int i = 0; i < batch_size; ++i) { - int64 length = scores.GetLength(i); + int64_t length = scores.GetLength(i); total_length += length; if (length > max_length) { max_length = length; @@ -182,7 +181,7 @@ class ConstrainedSequence : public OpKernel { Tensor *output; OP_REQUIRES_OK(context, context->allocate_output( 0, TensorShape({total_length}), &output)); - int32 *output_data = output->flat().data(); + int32_t* output_data = output->flat().data(); Tensor *offsets; OP_REQUIRES_OK(context, context->allocate_output( @@ -192,7 +191,7 @@ class ConstrainedSequence : public OpKernel { for (int batch = 0; batch < batch_size; ++batch) { int step_offset = offset_data[batch]; - int64 num_steps = scores.GetLength(batch); + int64_t num_steps = scores.GetLength(batch); offset_data[batch + 1] = step_offset + num_steps; if (use_viterbi_) { DoViterbiAnalysis(transition_weights, allowed_transitions, batch, @@ -207,18 +206,18 @@ class ConstrainedSequence : public OpKernel { private: // Perform Viterbi analysis on a single batch item. void DoViterbiAnalysis( - const tensorflow::TTypes::Matrix &transition_weights, - const tensorflow::TTypes::Matrix &allowed_transitions, - const int batch, const ScoreAccessor &scores, int32 *output_data) { + const tensorflow::TTypes::Matrix& transition_weights, + const tensorflow::TTypes::Matrix& allowed_transitions, + const int batch, const ScoreAccessor& scores, int32_t* output_data) { ViterbiAnalysis(scores, transition_weights, allowed_transitions, batch, use_log_space_, use_start_end_states_, output_data); } // Perform a greedy analysis on a single batch item. void DoGreedyAnalysis( - const tensorflow::TTypes::Matrix &transition_weights, - const tensorflow::TTypes::Matrix &allowed_transitions, - int batch, const ScoreAccessor &scores, int32 *output_data) { + const tensorflow::TTypes::Matrix& transition_weights, + const tensorflow::TTypes::Matrix& allowed_transitions, + int batch, const ScoreAccessor& scores, int32_t* output_data) { GreedyAnalysis(scores, transition_weights, allowed_transitions, batch, use_log_space_, use_start_end_states_, output_data); } @@ -251,8 +250,8 @@ class ConstrainedSequence : public OpKernel { .TypeConstraint("Tsplits"), \ ConstrainedSequence) -REGISTER_KERNELS(int32); -REGISTER_KERNELS(int64); +REGISTER_KERNELS(int32_t); +REGISTER_KERNELS(int64_t); #undef REGISTER_KERNELS diff --git a/tensorflow_text/core/kernels/constrained_sequence_kernel_input_validation_test.cc b/tensorflow_text/core/kernels/constrained_sequence_kernel_input_validation_test.cc index 343d2d142..023fff2b1 100644 --- a/tensorflow_text/core/kernels/constrained_sequence_kernel_input_validation_test.cc +++ b/tensorflow_text/core/kernels/constrained_sequence_kernel_input_validation_test.cc @@ -75,8 +75,8 @@ TEST_F(ConstrainedSequenceInputValidationTest, WorksWithInt64InputLengths) { }}); // Add the sequence_lengths input. - std::vector input_lengths({1, 1, 1}); - AddInputFromArray(TensorShape({3}), input_lengths); + std::vector input_lengths({1, 1, 1}); + AddInputFromArray(TensorShape({3}), input_lengths); // Add the allowed_transitions input. AddInputFromArray(TensorShape({5, 5}), @@ -99,8 +99,8 @@ TEST_F(ConstrainedSequenceInputValidationTest, WorksWithInt64InputLengths) { // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); diff --git a/tensorflow_text/core/kernels/disjoint_set_forest_test.cc b/tensorflow_text/core/kernels/disjoint_set_forest_test.cc index f63d92f08..2e4a26222 100644 --- a/tensorflow_text/core/kernels/disjoint_set_forest_test.cc +++ b/tensorflow_text/core/kernels/disjoint_set_forest_test.cc @@ -62,10 +62,10 @@ class DisjointSetForestTest : public ::testing::Test { }; using Forests = ::testing::Types< - DisjointSetForest, DisjointSetForest, - DisjointSetForest, DisjointSetForest, - DisjointSetForest, DisjointSetForest, - DisjointSetForest, DisjointSetForest>; + DisjointSetForest, DisjointSetForest, + DisjointSetForest, DisjointSetForest, + DisjointSetForest, DisjointSetForest, + DisjointSetForest, DisjointSetForest>; TYPED_TEST_SUITE(DisjointSetForestTest, Forests); TYPED_TEST(DisjointSetForestTest, DefaultEmpty) { @@ -111,12 +111,13 @@ TYPED_TEST(DisjointSetForestTest, Populated) { // merged set can be controlled. class DisjointSetForestNoUnionByRankTest : public ::testing::Test { protected: - using Forest = DisjointSetForest; + using Forest = DisjointSetForest; // Expects that the roots of the |forest| match |expected_roots|. - void ExpectRoots(const std::vector &expected_roots, Forest *forest) { + void ExpectRoots(const std::vector& expected_roots, + Forest* forest) { ASSERT_EQ(expected_roots.size(), forest->size()); - for (uint32 i = 0; i < forest->size(); ++i) { + for (uint32_t i = 0; i < forest->size(); ++i) { EXPECT_EQ(expected_roots[i], forest->FindRoot(i)); } } diff --git a/tensorflow_text/core/kernels/exp_greedy_constrained_sequence_kernel_test.cc b/tensorflow_text/core/kernels/exp_greedy_constrained_sequence_kernel_test.cc index 0c91e24be..753f930a3 100644 --- a/tensorflow_text/core/kernels/exp_greedy_constrained_sequence_kernel_test.cc +++ b/tensorflow_text/core/kernels/exp_greedy_constrained_sequence_kernel_test.cc @@ -96,8 +96,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -142,8 +142,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -183,8 +183,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // The sequence's highest score is 2, but OUT->2 is not ok, so it's 1. // Validate the output. - std::vector expected_transitions({1}); - std::vector expected_offsets({0, 1}); + std::vector expected_transitions({1}); + std::vector expected_offsets({0, 1}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -228,8 +228,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, int64inint32out) { // The third sequence's highest score is 0, which is ok. // Validate the output. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -274,8 +274,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, TwoDimensionalSequenceLengths) { // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -321,8 +321,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // The second sequence's highest score is 3, OUT->3 is OK and 3->OUT is OK. // The third sequence's highest score is 0, OUT->0 is OK and 0->OUT is OK. // Validate the output. - std::vector expected_transitions({0, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -364,8 +364,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // 2: {0.1, 4.5, 5.5, 5.0} (max is 2) // 3: {10.0, 12.0, 1.5, 4.0} (max is 1) // Validate the output. - std::vector expected_transitions({3, 2, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 2, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -407,8 +407,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // 2: {0.1, 4.5, 5.5, 5.0} (max is 2) // 3: {10.0, 12.0, 1.5, 4.0} (max is 1) // Validate the output. - std::vector expected_transitions({3, 2, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 2, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -451,8 +451,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // 2: {0.1, 4.5, 5.5, 0.5} (max is 2) // 3: {10.0, 12.0, 1.5, 0.4} (max is 1) // Validate the output. - std::vector expected_transitions({2, 2, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({2, 2, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -505,8 +505,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // 2: {.5, 4.5, 5.5, 2.5} (max is 2) // 3: {50.0, 12.0, 1.5,2.0} (max is 0) // Validate the output. - std::vector expected_transitions({0, 2, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 2, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -556,8 +556,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // 2: {0.1, 4.5, 5.5, 5.0} (max is 2). OUT->2 is OK. // 3: {10.0, 12.0, 1.5, 4.0} (max is 1). OUT->1 is not OK, so go with 0. // Note that X->OUT is set to always be OK here. - std::vector expected_transitions({3, 2, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 2, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -619,8 +619,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // 3: In state 0, so use row 0 in the weight tensor. // Weights are {0.5, 5.5, 0.5, 5}; 0->1 is OK but 1->OUT is not, so 3. - std::vector expected_transitions({3, 2, 2, 0, 0, 3}); - std::vector expected_offsets({0, 2, 4, 6}); + std::vector expected_transitions({3, 2, 2, 0, 0, 3}); + std::vector expected_offsets({0, 2, 4, 6}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -680,8 +680,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // 3: In state 0, so use row 0 in the weight tensor. // Weights are {0.5, 5.5, 0.5, 5}; 0->1 is OK but 1->OUT is not, so 3. - std::vector expected_transitions({3, 2, 2, 0, 3}); - std::vector expected_offsets({0, 2, 3, 5}); + std::vector expected_transitions({3, 2, 2, 0, 3}); + std::vector expected_offsets({0, 2, 3, 5}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -721,8 +721,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, TF_ASSERT_OK(RunOpKernel()); - std::vector expected_transitions({3, 0, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 0, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -763,8 +763,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // In the case of a tie between weights, the higher state number wins; // if all weights are zero, the states should all be 3. - std::vector expected_transitions({3, 3, 3}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 3, 3}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -808,8 +808,8 @@ TEST_F(ExpGreedyConstrainedSequenceTest, // Validate the output. - std::vector expected_transitions({-1, -1, -1, -1, -1, -1}); - std::vector expected_offsets({0, 2, 4, 6}); + std::vector expected_transitions({-1, -1, -1, -1, -1, -1}); + std::vector expected_offsets({0, 2, 4, 6}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); diff --git a/tensorflow_text/core/kernels/exp_viterbi_constrained_sequence_kernel_test.cc b/tensorflow_text/core/kernels/exp_viterbi_constrained_sequence_kernel_test.cc index 49cfa02be..61502a739 100644 --- a/tensorflow_text/core/kernels/exp_viterbi_constrained_sequence_kernel_test.cc +++ b/tensorflow_text/core/kernels/exp_viterbi_constrained_sequence_kernel_test.cc @@ -96,8 +96,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -142,8 +142,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -183,8 +183,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // The sequence's highest score is 2, but OUT->2 is not ok, so it's 1. // Validate the output. - std::vector expected_transitions({1}); - std::vector expected_offsets({0, 1}); + std::vector expected_transitions({1}); + std::vector expected_offsets({0, 1}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -228,8 +228,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, int64inint32out) { // The third sequence's highest score is 0, which is ok. // Validate the output. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -274,8 +274,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, TwoDimensionalSequenceLengths) { // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -321,8 +321,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // The second sequence's highest score is 3, OUT->3 is OK and 3->OUT is OK. // The third sequence's highest score is 0, OUT->0 is OK and 0->OUT is OK. // Validate the output. - std::vector expected_transitions({0, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -364,8 +364,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // 2: {0.1, 4.5, 5.5, 5.0} (max is 2) // 3: {10.0, 12.0, 1.5, 4.0} (max is 1) // Validate the output. - std::vector expected_transitions({3, 2, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 2, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -407,8 +407,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // 2: {0.1, 4.5, 5.5, 5.0} (max is 2) // 3: {10.0, 12.0, 1.5, 4.0} (max is 1) // Validate the output. - std::vector expected_transitions({3, 2, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 2, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -451,8 +451,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // 2: {0.1, 4.5, 5.5, 0.5} (max is 2) // 3: {10.0, 12.0, 1.5, 0.4} (max is 1) // Validate the output. - std::vector expected_transitions({2, 2, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({2, 2, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -505,8 +505,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // 2: {.5, 4.5, 5.5, 2.5} (max is 2) // 3: {50.0, 12.0, 1.5,2.0} (max is 0) // Validate the output. - std::vector expected_transitions({0, 2, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 2, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -556,8 +556,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // 2: {0.1, 4.5, 5.5, 5.0} (max is 2). OUT->2 is OK. // 3: {10.0, 12.0, 1.5, 4.0} (max is 1). OUT->1 is not OK, so go with 0. // Note that X->OUT is set to always be OK here. - std::vector expected_transitions({3, 2, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 2, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -646,8 +646,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // 1->OUT is not valid, so final scores are [5, 0, 5, 50] for a final // state of 3 and a sequence of [0, 3]. - std::vector expected_transitions({3, 2, 3, 3, 0, 3}); - std::vector expected_offsets({0, 2, 4, 6}); + std::vector expected_transitions({3, 2, 3, 3, 0, 3}); + std::vector expected_offsets({0, 2, 4, 6}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -729,8 +729,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // 1->OUT is not valid, so final scores are [5, 0, 5, 50] for a final // state of 3 and a sequence of [0, 3]. - std::vector expected_transitions({3, 2, 2, 0, 3}); - std::vector expected_offsets({0, 2, 3, 5}); + std::vector expected_transitions({3, 2, 2, 0, 3}); + std::vector expected_offsets({0, 2, 3, 5}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -771,8 +771,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // In the case of a tie between weights, the higher state number wins; // if all weights are zero, the states should all be 3. - std::vector expected_transitions({3, 3, 3}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 3, 3}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -816,8 +816,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, // Validate the output. - std::vector expected_transitions({-1, -1, -1, -1, -1, -1}); - std::vector expected_offsets({0, 2, 4, 6}); + std::vector expected_transitions({-1, -1, -1, -1, -1, -1}); + std::vector expected_offsets({0, 2, 4, 6}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -899,8 +899,8 @@ TEST_F(ExpViterbiConstrainedSequenceTest, OutputsInt32RaggedTensor) { TF_ASSERT_OK(RunOpKernel()); - std::vector expected_transitions({3, 2, 2, 0, 3}); - std::vector expected_offsets({0, 2, 3, 5}); + std::vector expected_transitions({3, 2, 2, 0, 3}); + std::vector expected_offsets({0, 2, 3, 5}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); diff --git a/tensorflow_text/core/kernels/fast_bert_normalizer_kernel_template.h b/tensorflow_text/core/kernels/fast_bert_normalizer_kernel_template.h index ae795b53b..68d4fd0fc 100644 --- a/tensorflow_text/core/kernels/fast_bert_normalizer_kernel_template.h +++ b/tensorflow_text/core/kernels/fast_bert_normalizer_kernel_template.h @@ -164,7 +164,7 @@ absl::Status FastBertNormalizeOp::InvokeRealWork(InvokeContext* context) { // memory-mapped wrapper on `fast_bert_normalizer_model` tensor, and thus // Create() is very cheap. auto text_normalizer = FastBertNormalizer::Create( - fast_bert_normalizer_model->template Data().data()); + fast_bert_normalizer_model->template Data().data()); SH_RETURN_IF_ERROR(text_normalizer.status()); SH_ASSIGN_OR_RETURN( @@ -214,15 +214,15 @@ absl::Status FastBertNormalizeOp::InvokeRealWork(InvokeContext* context) { } if constexpr (kGetOffsets) { - SH_RETURN_IF_ERROR(this->template FillOutputTensor( + SH_RETURN_IF_ERROR(this->template FillOutputTensor( offsets, kOutputOffsets, context)); - SH_RETURN_IF_ERROR(this->template FillOutputTensor( + SH_RETURN_IF_ERROR(this->template FillOutputTensor( row_splits, kOutputRowSplitsOfOffsets, context)); } else { - SH_RETURN_IF_ERROR(this->template FillOutputTensor( + SH_RETURN_IF_ERROR(this->template FillOutputTensor( offsets, kOutputOffsets, context)); row_splits.resize(1+values_vec.Dim(0)); - SH_RETURN_IF_ERROR(this->template FillOutputTensor( + SH_RETURN_IF_ERROR(this->template FillOutputTensor( row_splits, kOutputRowSplitsOfOffsets, context)); } return absl::OkStatus(); diff --git a/tensorflow_text/core/kernels/fast_wordpiece_tokenizer_kernel_template.h b/tensorflow_text/core/kernels/fast_wordpiece_tokenizer_kernel_template.h index efc26197a..bb16e3d84 100644 --- a/tensorflow_text/core/kernels/fast_wordpiece_tokenizer_kernel_template.h +++ b/tensorflow_text/core/kernels/fast_wordpiece_tokenizer_kernel_template.h @@ -147,7 +147,7 @@ absl::Status FastWordpieceTokenizeWithOffsetsOp::Invoke( // Create() is very cheap. auto fast_wordpiece_tokenizer = ::tensorflow::text::FastWordpieceTokenizer::Create( - wp_model->template Data().data()); + wp_model->template Data().data()); SH_RETURN_IF_ERROR(fast_wordpiece_tokenizer.status()); // TODO(xysong): Optimize based on which information below is requested. @@ -180,13 +180,13 @@ absl::Status FastWordpieceTokenizeWithOffsetsOp::Invoke( SH_RETURN_IF_ERROR(this->template FillOutputTensor( subwords, kOutputSubwords, context)); - SH_RETURN_IF_ERROR(this->template FillOutputTensor( + SH_RETURN_IF_ERROR(this->template FillOutputTensor( subword_ids, kOutputIds, context)); - SH_RETURN_IF_ERROR(this->template FillOutputTensor( + SH_RETURN_IF_ERROR(this->template FillOutputTensor( row_splits, kOutputRowSplits, context)); - SH_RETURN_IF_ERROR(this->template FillOutputTensor( + SH_RETURN_IF_ERROR(this->template FillOutputTensor( begin_offset, kStartValues, context)); - SH_RETURN_IF_ERROR(this->template FillOutputTensor( + SH_RETURN_IF_ERROR(this->template FillOutputTensor( end_offset, kEndValues, context)); return absl::OkStatus(); @@ -311,7 +311,7 @@ absl::Status FastWordpieceDetokenizeOp::Invoke(InvokeContext* context) { SH_ASSIGN_OR_RETURN(const auto input_row_splits, context->GetInput(kInputRowSplits)); - const auto& row_splits_vec = input_row_splits->template As(); + const auto& row_splits_vec = input_row_splits->template As(); SH_ASSIGN_OR_RETURN(const auto wp_model, context->GetInput(kWpModel)); // OK to create on every call because FastWordpieceTokenizer is a @@ -319,7 +319,7 @@ absl::Status FastWordpieceDetokenizeOp::Invoke(InvokeContext* context) { // Create() is very cheap. auto fast_wordpiece_tokenizer = ::tensorflow::text::FastWordpieceTokenizer::Create( - wp_model->template Data().data()); + wp_model->template Data().data()); SH_RETURN_IF_ERROR(fast_wordpiece_tokenizer.status()); std::vector sentences; diff --git a/tensorflow_text/core/kernels/log_greedy_constrained_sequence_kernel_test.cc b/tensorflow_text/core/kernels/log_greedy_constrained_sequence_kernel_test.cc index 6d9d89054..5b3e6e54d 100644 --- a/tensorflow_text/core/kernels/log_greedy_constrained_sequence_kernel_test.cc +++ b/tensorflow_text/core/kernels/log_greedy_constrained_sequence_kernel_test.cc @@ -96,8 +96,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -142,8 +142,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -183,8 +183,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // The sequence's highest score is 2, but OUT->2 is not ok, so it's 1. // Validate the output. - std::vector expected_transitions({1}); - std::vector expected_offsets({0, 1}); + std::vector expected_transitions({1}); + std::vector expected_offsets({0, 1}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -228,8 +228,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, int64inint32out) { // The third sequence's highest score is 0, which is ok. // Validate the output. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -274,8 +274,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, TwoDimensionalSequenceLengths) { // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -321,8 +321,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // The second sequence's highest score is 3, OUT->3 is OK and 3->OUT is OK. // The third sequence's highest score is 0, OUT->0 is OK and 0->OUT is OK. // Validate the output. - std::vector expected_transitions({0, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -364,8 +364,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // 2: {1.1, 9.5, 11.5, 6.0} (max is 2) // 3: {100.1, 24.5, 3.5, 5.0} (max is 0) // Validate the output. - std::vector expected_transitions({0, 2, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 2, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -407,8 +407,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // 2: {1.1, 9.5, 11.5, 6.0} (max is 2) // 3: {100.1, 24.5, 3.5, 5.0} (max is 0) // Validate the output. - std::vector expected_transitions({0, 2, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 2, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -451,8 +451,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // 2: {1.1, 9.5, 11.5, 6.0} (max is 2) // 3: {100.1, 24.5, 3.5, 5.0} (max is 0) // Validate the output. - std::vector expected_transitions({0, 2, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 2, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -502,8 +502,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // 2: {1.1, 9.5, 11.5, 6.0} (max is 2, but 2->NUL is not OK, so 1.) // 3: {100.1, 24.5, 3.5, 5.0} (max is 0, but NUL->0 is not OK, so 1.) // Validate the output. - std::vector expected_transitions({3, 1, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 1, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -565,8 +565,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // 0. 3: In state 0, so use row 0 in the weight tensor. Weights are // {1.5, 11.5, 1.5, 11}; 0->1 is OK but 1->OUT is not, so 3. - std::vector expected_transitions({2, 2, 2, 0, 1, 3}); - std::vector expected_offsets({0, 2, 4, 6}); + std::vector expected_transitions({2, 2, 2, 0, 1, 3}); + std::vector expected_offsets({0, 2, 4, 6}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -626,8 +626,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // 3: In state 0, so use row 0 in the weight tensor. // Weights are {1.5, 11.5, 1.5, 11}; 0->1 is OK but 1->OUT is not, so 3. - std::vector expected_transitions({2, 2, 2, 1, 3}); - std::vector expected_offsets({0, 2, 3, 5}); + std::vector expected_transitions({2, 2, 2, 1, 3}); + std::vector expected_offsets({0, 2, 3, 5}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -667,8 +667,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, TF_ASSERT_OK(RunOpKernel()); - std::vector expected_transitions({3, 0, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 0, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -708,8 +708,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // Because all weights are zero, the max values should be the max of the // scores. - std::vector expected_transitions({0, 2, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 2, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -753,8 +753,8 @@ TEST_F(LogGreedyConstrainedSequenceTest, // Validate the output. - std::vector expected_transitions({-1, -1, -1, -1, -1, -1}); - std::vector expected_offsets({0, 2, 4, 6}); + std::vector expected_transitions({-1, -1, -1, -1, -1, -1}); + std::vector expected_offsets({0, 2, 4, 6}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); diff --git a/tensorflow_text/core/kernels/log_viterbi_constrained_sequence_kernel_test.cc b/tensorflow_text/core/kernels/log_viterbi_constrained_sequence_kernel_test.cc index 7e444a496..cf8e6a708 100644 --- a/tensorflow_text/core/kernels/log_viterbi_constrained_sequence_kernel_test.cc +++ b/tensorflow_text/core/kernels/log_viterbi_constrained_sequence_kernel_test.cc @@ -96,8 +96,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -142,8 +142,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -183,8 +183,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, // The sequence's highest score is 2, but OUT->2 is not ok, so it's 1. // Validate the output. - std::vector expected_transitions({1}); - std::vector expected_offsets({0, 1}); + std::vector expected_transitions({1}); + std::vector expected_offsets({0, 1}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -228,8 +228,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, int64inint32out) { // The third sequence's highest score is 0, which is ok. // Validate the output. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -274,8 +274,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, TwoDimensionalSequenceLengths) { // The third sequence's highest score is 0, which is ok. // Validate the output. - std::vector expected_transitions({1, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({1, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -321,8 +321,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, // The second sequence's highest score is 3, OUT->3 is OK and 3->OUT is OK. // The third sequence's highest score is 0, OUT->0 is OK and 0->OUT is OK. // Validate the output. - std::vector expected_transitions({0, 3, 0}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 3, 0}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -364,8 +364,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, // 2: {11.0, 14.0, 14.0, 6.0} (max is 2, due to tiebreaker.) // 3: {-2.0, 8.0, 6.0, 5.0} (max is 1) // Validate the output. - std::vector expected_transitions({0, 2, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 2, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -407,8 +407,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, // 2: {11.0, 14.0, 14.0, 6.0} (max is 2, due to tiebreaker.) // 3: {-2.0, 8.0, 6.0, 5.0} (max is 1) // Validate the output. - std::vector expected_transitions({0, 2, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({0, 2, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -451,8 +451,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, // 2: {11.0, 14.0, 14.0, 6.0} (max is 2, due to tiebreaker.) // 3: {-2.0, 8.0, 6.0, 5.0} (max is 1) // Validate the output. - std::vector expected_transitions({2, 2, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({2, 2, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -502,8 +502,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, // 2: {11.0, 14.0, 14.0, 6.0} (max is 2, due to tiebreaker.) // 3: {-2.0, 8.0, 6.0, 5.0} (max is 1) // Validate the output. - std::vector expected_transitions({2, 2, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({2, 2, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -601,8 +601,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, // final state of [1] with a sequence of [3->1]. // - std::vector expected_transitions({2, 0, 3, 1}); - std::vector expected_offsets({0, 2, 4}); + std::vector expected_transitions({2, 0, 3, 1}); + std::vector expected_offsets({0, 2, 4}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -683,8 +683,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, // final state of [1] with a sequence of [3->1]. // - std::vector expected_transitions({0, 3, 1}); - std::vector expected_offsets({0, 1, 3}); + std::vector expected_transitions({0, 3, 1}); + std::vector expected_offsets({0, 1, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -724,8 +724,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, TF_ASSERT_OK(RunOpKernel()); - std::vector expected_transitions({3, 0, 1}); - std::vector expected_offsets({0, 1, 2, 3}); + std::vector expected_transitions({3, 0, 1}); + std::vector expected_offsets({0, 1, 2, 3}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); @@ -769,8 +769,8 @@ TEST_F(LogViterbiConstrainedSequenceTest, // Validate the output. - std::vector expected_transitions({-1, -1, -1, -1, -1, -1}); - std::vector expected_offsets({0, 2, 4, 6}); + std::vector expected_transitions({-1, -1, -1, -1, -1, -1}); + std::vector expected_offsets({0, 2, 4, 6}); // Validate the output. EXPECT_THAT(*GetOutput(0), VectorEq(expected_transitions)); diff --git a/tensorflow_text/core/kernels/mst_op_kernels.cc b/tensorflow_text/core/kernels/mst_op_kernels.cc index 01eb5954b..ed42581e3 100644 --- a/tensorflow_text/core/kernels/mst_op_kernels.cc +++ b/tensorflow_text/core/kernels/mst_op_kernels.cc @@ -53,8 +53,8 @@ class MaxSpanningTreeOpKernel : public tensorflow::OpKernel { scores_tensor.shape().DebugString())); // Batch size and input dimension (B and M in the op docstring). - const int64 batch_size = scores_tensor.shape().dim_size(0); - const int64 input_dim = scores_tensor.shape().dim_size(1); + const int64_t batch_size = scores_tensor.shape().dim_size(0); + const int64_t input_dim = scores_tensor.shape().dim_size(1); // Check shapes. const tensorflow::TensorShape shape_b({batch_size}); @@ -81,18 +81,19 @@ class MaxSpanningTreeOpKernel : public tensorflow::OpKernel { &argmax_sources_tensor)); // Acquire shaped and typed references. - const BatchedSizes num_nodes_b = num_nodes_tensor.vec(); + const BatchedSizes num_nodes_b = num_nodes_tensor.vec(); const BatchedScores scores_bxmxm = scores_tensor.tensor(); BatchedMaxima max_scores_b = max_scores_tensor->vec(); - BatchedSources argmax_sources_bxm = argmax_sources_tensor->matrix(); + BatchedSources argmax_sources_bxm = + argmax_sources_tensor->matrix(); // Solve the batch of MST problems in parallel. Set a high cycles per unit // to encourage finer sharding. - constexpr int64 kCyclesPerUnit = 1000 * 1000 * 1000; + constexpr int64_t kCyclesPerUnit = 1000 * 1000 * 1000; std::vector statuses(batch_size); context->device()->tensorflow_cpu_worker_threads()->workers->ParallelFor( - batch_size, kCyclesPerUnit, [&](int64 begin, int64 end) { - for (int64 problem = begin; problem < end; ++problem) { + batch_size, kCyclesPerUnit, [&](int64_t begin, int64_t end) { + for (int64_t problem = begin; problem < end; ++problem) { statuses[problem] = RunSolver(problem, num_nodes_b, scores_bxmxm, max_scores_b, argmax_sources_bxm); } @@ -103,10 +104,10 @@ class MaxSpanningTreeOpKernel : public tensorflow::OpKernel { } private: - using BatchedSizes = typename tensorflow::TTypes::ConstVec; + using BatchedSizes = typename tensorflow::TTypes::ConstVec; using BatchedScores = typename tensorflow::TTypes::ConstTensor; using BatchedMaxima = typename tensorflow::TTypes::Vec; - using BatchedSources = typename tensorflow::TTypes::Matrix; + using BatchedSources = typename tensorflow::TTypes::Matrix; // Solves for the maximum spanning tree of the digraph defined by the values // at index |problem| in |num_nodes_b| and |scores_bxmxm|. On success, sets @@ -116,8 +117,8 @@ class MaxSpanningTreeOpKernel : public tensorflow::OpKernel { BatchedScores scores_bxmxm, BatchedMaxima max_scores_b, BatchedSources argmax_sources_bxm) const { // Check digraph size overflow. - const int32 num_nodes = num_nodes_b(problem); - const int32 input_dim = argmax_sources_bxm.dimension(1); + const int32_t num_nodes = num_nodes_b(problem); + const int32_t input_dim = argmax_sources_bxm.dimension(1); if (num_nodes > input_dim) { return tensorflow::errors::InvalidArgument( "number of nodes in digraph ", problem, @@ -161,7 +162,7 @@ class MaxSpanningTreeOpKernel : public tensorflow::OpKernel { max_scores_b(problem) = max_score; // Pad the source list with -1. - for (int32 i = num_nodes; i < input_dim; ++i) { + for (int32_t i = num_nodes; i < input_dim; ++i) { argmax_sources_bxm(problem, i) = -1; } @@ -175,16 +176,16 @@ class MaxSpanningTreeOpKernel : public tensorflow::OpKernel { // Use Index=uint16, which allows digraphs containing up to 32,767 nodes. REGISTER_KERNEL_BUILDER(Name("MaxSpanningTree") .Device(tensorflow::DEVICE_CPU) - .TypeConstraint("T"), - MaxSpanningTreeOpKernel); + .TypeConstraint("T"), + MaxSpanningTreeOpKernel); REGISTER_KERNEL_BUILDER(Name("MaxSpanningTree") .Device(tensorflow::DEVICE_CPU) .TypeConstraint("T"), - MaxSpanningTreeOpKernel); + MaxSpanningTreeOpKernel); REGISTER_KERNEL_BUILDER(Name("MaxSpanningTree") .Device(tensorflow::DEVICE_CPU) .TypeConstraint("T"), - MaxSpanningTreeOpKernel); + MaxSpanningTreeOpKernel); } // namespace text } // namespace tensorflow diff --git a/tensorflow_text/core/kernels/mst_solver_random_comparison_test.cc b/tensorflow_text/core/kernels/mst_solver_random_comparison_test.cc index 9896801b5..2feffede6 100644 --- a/tensorflow_text/core/kernels/mst_solver_random_comparison_test.cc +++ b/tensorflow_text/core/kernels/mst_solver_random_comparison_test.cc @@ -35,40 +35,41 @@ namespace text { using ::testing::Contains; // Returns the random seed, or 0 for a weak random seed. -int64 GetSeed() { return absl::GetFlag(FLAGS_seed); } +int64_t GetSeed() { return absl::GetFlag(FLAGS_seed); } // Returns the number of trials to run for each random comparison. -int64 GetNumTrials() { return absl::GetFlag(FLAGS_num_trials); } +int64_t GetNumTrials() { return absl::GetFlag(FLAGS_num_trials); } // Testing rig. Runs a comparison between a brute-force MST solver and the // MstSolver<> on random digraphs. When the first test parameter is true, // solves for forests instead of trees. The second test parameter defines the // size of the test digraph. class MstSolverRandomComparisonTest - : public ::testing::TestWithParam<::testing::tuple> { + : public ::testing::TestWithParam<::testing::tuple> { protected: // Use integer scores so score comparisons are exact. - using Solver = MstSolver; + using Solver = MstSolver; // An array providing a source node for each node. Roots are self-loops. using SourceList = SpanningTreeIterator::SourceList; // A row-major n x n matrix whose i,j entry gives the score of the arc from i // to j, and whose i,i entry gives the score of selecting i as a root. - using ScoreMatrix = std::vector; + using ScoreMatrix = std::vector; // Returns true if this should be a forest. bool forest() const { return ::testing::get<0>(GetParam()); } // Returns the number of nodes for digraphs. - uint32 num_nodes() const { return ::testing::get<1>(GetParam()); } + uint32_t num_nodes() const { return ::testing::get<1>(GetParam()); } // Returns the score of the arcs in |sources| based on the |scores|. - int32 ScoreArcs(const ScoreMatrix &scores, const SourceList &sources) const { + int32_t ScoreArcs(const ScoreMatrix& scores, + const SourceList& sources) const { CHECK_EQ(num_nodes() * num_nodes(), scores.size()); - int32 score = 0; - for (uint32 target = 0; target < num_nodes(); ++target) { - const uint32 source = sources[target]; + int32_t score = 0; + for (uint32_t target = 0; target < num_nodes(); ++target) { + const uint32_t source = sources[target]; score += scores[target + source * num_nodes()]; } return score; @@ -77,14 +78,14 @@ class MstSolverRandomComparisonTest // Returns the score of the maximum spanning tree (or forest, if the first // test parameter is true) of the dense digraph defined by the |scores|, and // sets |argmax_trees| to contain all maximal trees. - int32 RunBruteForceMstSolver(const ScoreMatrix &scores, - std::set *argmax_trees) { + int32_t RunBruteForceMstSolver(const ScoreMatrix& scores, + std::set* argmax_trees) { CHECK_EQ(num_nodes() * num_nodes(), scores.size()); - int32 max_score; + int32_t max_score; argmax_trees->clear(); - iterator_.ForEachTree(num_nodes(), [&](const SourceList &sources) { - const int32 score = ScoreArcs(scores, sources); + iterator_.ForEachTree(num_nodes(), [&](const SourceList& sources) { + const int32_t score = ScoreArcs(scores, sources); if (argmax_trees->empty() || max_score < score) { max_score = score; argmax_trees->clear(); @@ -98,14 +99,14 @@ class MstSolverRandomComparisonTest } // As above, but uses the |solver_| and extracts only one |argmax_tree|. - int32 RunMstSolver(const ScoreMatrix &scores, SourceList *argmax_tree) { + int32_t RunMstSolver(const ScoreMatrix& scores, SourceList* argmax_tree) { CHECK_EQ(num_nodes() * num_nodes(), scores.size()); TF_CHECK_OK(solver_.Init(forest(), num_nodes())); // Add all roots and arcs. - for (uint32 source = 0; source < num_nodes(); ++source) { - for (uint32 target = 0; target < num_nodes(); ++target) { - const int32 score = scores[target + source * num_nodes()]; + for (uint32_t source = 0; source < num_nodes(); ++source) { + for (uint32_t target = 0; target < num_nodes(); ++target) { + const int32_t score = scores[target + source * num_nodes()]; if (source == target) { solver_.AddRoot(target, score); } else { @@ -123,7 +124,8 @@ class MstSolverRandomComparisonTest // Returns a random ScoreMatrix spanning num_nodes() nodes. ScoreMatrix RandomScores() { ScoreMatrix scores(num_nodes() * num_nodes()); - for (int32 &value : scores) value = static_cast(prng_() % 201) - 100; + for (int32_t& value : scores) + value = static_cast(prng_() % 201) - 100; return scores; } @@ -132,7 +134,7 @@ class MstSolverRandomComparisonTest void RunComparison() { // Seed the PRNG, possibly non-deterministically. Log the seed value so the // test results can be reproduced, even when the seed is non-deterministic. - uint32 seed = GetSeed(); + uint32_t seed = GetSeed(); if (seed == 0) seed = time(nullptr); prng_.seed(seed); LOG(INFO) << "seed = " << seed; @@ -142,11 +144,12 @@ class MstSolverRandomComparisonTest const ScoreMatrix scores = RandomScores(); std::set expected_argmax_trees; - const int32 expected_max_score = + const int32_t expected_max_score = RunBruteForceMstSolver(scores, &expected_argmax_trees); SourceList actual_argmax_tree; - const int32 actual_max_score = RunMstSolver(scores, &actual_argmax_tree); + const int32_t actual_max_score = + RunMstSolver(scores, &actual_argmax_tree); // In case of ties, MstSolver will find a maximal spanning tree, but we // don't know which one. @@ -168,7 +171,7 @@ class MstSolverRandomComparisonTest INSTANTIATE_TEST_SUITE_P(AllowForest, MstSolverRandomComparisonTest, ::testing::Combine(::testing::Bool(), - ::testing::Range(1, 9))); + ::testing::Range(1, 9))); TEST_P(MstSolverRandomComparisonTest, Comparison) { RunComparison(); } diff --git a/tensorflow_text/core/kernels/mst_solver_test.cc b/tensorflow_text/core/kernels/mst_solver_test.cc index 782f7817e..9a57046df 100644 --- a/tensorflow_text/core/kernels/mst_solver_test.cc +++ b/tensorflow_text/core/kernels/mst_solver_test.cc @@ -84,9 +84,9 @@ class MstSolverTest : public ::testing::Test { }; using Solvers = - ::testing::Types, MstSolver, - MstSolver, MstSolver, - MstSolver>; + ::testing::Types, MstSolver, + MstSolver, MstSolver, + MstSolver>; TYPED_TEST_SUITE(MstSolverTest, Solvers); TYPED_TEST(MstSolverTest, FailIfNoNodes) { diff --git a/tensorflow_text/core/kernels/normalize_kernels.cc b/tensorflow_text/core/kernels/normalize_kernels.cc index e011a4629..859b473ed 100644 --- a/tensorflow_text/core/kernels/normalize_kernels.cc +++ b/tensorflow_text/core/kernels/normalize_kernels.cc @@ -59,9 +59,9 @@ class CaseFoldUTF8Op : public tensorflow::OpKernel { icu_error.errorName(), ": Could not retrieve ICU NFKC_CaseFold normalizer"))); - for (int64 i = 0; i < input_vec.size(); ++i) { - string output_text; - icu::StringByteSink byte_sink(&output_text); + for (int64_t i = 0; i < input_vec.size(); ++i) { + std::string output_text; + icu::StringByteSink byte_sink(&output_text); const auto& input = input_vec(i); nfkc_cf->normalizeUTF8(0, icu::StringPiece(input.data(), input.size()), byte_sink, nullptr, icu_error); @@ -78,9 +78,9 @@ REGISTER_KERNEL_BUILDER(Name("CaseFoldUTF8").Device(tensorflow::DEVICE_CPU), namespace { -string GetNormalizationForm(OpKernelConstruction* context) { - string normalization_form; - ([=](string* c) -> void { +std::string GetNormalizationForm(OpKernelConstruction* context) { + std::string normalization_form; + ([=](std::string* c) -> void { OP_REQUIRES_OK(context, context->GetAttr("normalization_form", c)); })(&normalization_form); return absl::AsciiStrToUpper(normalization_form); @@ -137,9 +137,9 @@ class NormalizeUTF8Op : public tensorflow::OpKernel { "Unknown normalization form requrested: ", normalization_form_))); } - for (int64 i = 0; i < input_vec.size(); ++i) { - string output_text; - icu::StringByteSink byte_sink(&output_text); + for (int64_t i = 0; i < input_vec.size(); ++i) { + std::string output_text; + icu::StringByteSink byte_sink(&output_text); const auto& input = input_vec(i); normalizer->normalizeUTF8(0, icu::StringPiece(input.data(), input.size()), byte_sink, nullptr, icu_error); @@ -153,7 +153,7 @@ class NormalizeUTF8Op : public tensorflow::OpKernel { } private: - string normalization_form_; + std::string normalization_form_; }; REGISTER_KERNEL_BUILDER(Name("NormalizeUTF8").Device(tensorflow::DEVICE_CPU), @@ -168,7 +168,7 @@ namespace { // reconstruct the icu::Edits object from the serialized `changes` string when // the variant is at the graph input. struct OffsetMapVariant { - string changes; + std::string changes; icu::Edits edits_; std::string TypeName() const { return "(anonymous)::OffsetMapVariant"; } @@ -185,18 +185,18 @@ void OffsetMapVariant::Encode(tensorflow::VariantTensorData* data) const { change->set_old_length(it.oldLength()); change->set_new_length(it.newLength()); } - string changes_str = changes.SerializeAsString(); + std::string changes_str = changes.SerializeAsString(); data->set_metadata(changes_str); } bool OffsetMapVariant::Decode(const tensorflow::VariantTensorData& data) { - string serialized; + std::string serialized; data.get_metadata(&serialized); EditChanges changes; changes.ParseFromString(serialized); icu::Edits edit; icu::ErrorCode icu_error; - for (int64 j = 0; j < changes.change_size(); ++j) { + for (int64_t j = 0; j < changes.change_size(); ++j) { auto* change = changes.mutable_change(j); int old_length = change->old_length(); int new_length = change->new_length(); @@ -268,11 +268,11 @@ class NormalizeUTF8WithOffsetsMapOp : public tensorflow::OpKernel { normalization_form_))); } - for (int64 i = 0; i < input_vec.size(); ++i) { + for (int64_t i = 0; i < input_vec.size(); ++i) { OffsetMapVariant variant; - string output_text; + std::string output_text; icu::Edits edits; - icu::StringByteSink byte_sink(&output_text); + icu::StringByteSink byte_sink(&output_text); const auto& input = input_vec(i); normalizer->normalizeUTF8(0, icu::StringPiece(input.data(), input.size()), byte_sink, &edits, icu_error); @@ -289,7 +289,7 @@ class NormalizeUTF8WithOffsetsMapOp : public tensorflow::OpKernel { } private: - string normalization_form_; + std::string normalization_form_; }; REGISTER_KERNEL_BUILDER( @@ -307,18 +307,18 @@ class FindSourceOffsetsOp : public tensorflow::OpKernel { const tensorflow::Tensor& input_offsets_values = context->input(1); const tensorflow::Tensor& input_offsets_splits = context->input(2); - const auto& input_offsets_values_vec = input_offsets_values.flat(); + const auto& input_offsets_values_vec = input_offsets_values.flat(); const auto& input_offsets_splits_vec = input_offsets_splits.flat(); const auto& edits_vec = edits_values.flat(); icu::ErrorCode icu_error; - int64 cur_split_index_begin = 0; - int64 cur_split_index_end = 0; - std::vector output_offsets_values(input_offsets_values_vec.size()); - int64 idx_edits = 0; - int64 idx_output = 0; - for (int64 i = 0; i < input_offsets_splits_vec.size() - 1; ++i) { + int64_t cur_split_index_begin = 0; + int64_t cur_split_index_end = 0; + std::vector output_offsets_values(input_offsets_values_vec.size()); + int64_t idx_edits = 0; + int64_t idx_output = 0; + for (int64_t i = 0; i < input_offsets_splits_vec.size() - 1; ++i) { cur_split_index_begin = input_offsets_splits_vec(i); cur_split_index_end = input_offsets_splits_vec(i + 1); if (cur_split_index_begin == cur_split_index_end) { @@ -331,7 +331,7 @@ class FindSourceOffsetsOp : public tensorflow::OpKernel { auto iter = edits_vec(idx_edits++) .get() ->edits_.getFineChangesIterator(); - for (int64 j = cur_split_index_begin; j < cur_split_index_end; ++j) { + for (int64_t j = cur_split_index_begin; j < cur_split_index_end; ++j) { output_offsets_values[idx_output++] = iter.sourceIndexFromDestinationIndex(input_offsets_values_vec(j), icu_error); @@ -342,16 +342,16 @@ class FindSourceOffsetsOp : public tensorflow::OpKernel { "Input offset tensor dimension did not match the offset " "map dimension.")); - int64 output_offsets_values_size = output_offsets_values.size(); + int64_t output_offsets_values_size = output_offsets_values.size(); Tensor* output_offsets_values_tensor = nullptr; OP_REQUIRES_OK(context, context->allocate_output( "output_offsets_values", TensorShape({output_offsets_values_size}), &output_offsets_values_tensor)); auto output_offsets_values_data = - output_offsets_values_tensor->flat().data(); + output_offsets_values_tensor->flat().data(); memcpy(output_offsets_values_data, output_offsets_values.data(), - output_offsets_values_size * sizeof(int64)); + output_offsets_values_size * sizeof(int64_t)); } private: @@ -360,11 +360,11 @@ class FindSourceOffsetsOp : public tensorflow::OpKernel { REGISTER_KERNEL_BUILDER(Name("FindSourceOffsets") .Device(tensorflow::DEVICE_CPU) - .TypeConstraint("Tsplits"), - FindSourceOffsetsOp); + .TypeConstraint("Tsplits"), + FindSourceOffsetsOp); REGISTER_KERNEL_BUILDER(Name("FindSourceOffsets") .Device(tensorflow::DEVICE_CPU) - .TypeConstraint("Tsplits"), - FindSourceOffsetsOp); + .TypeConstraint("Tsplits"), + FindSourceOffsetsOp); } // namespace text } // namespace tensorflow diff --git a/tensorflow_text/core/kernels/phrase_tokenizer_kernel_template.h b/tensorflow_text/core/kernels/phrase_tokenizer_kernel_template.h index 67807a768..22ac29871 100644 --- a/tensorflow_text/core/kernels/phrase_tokenizer_kernel_template.h +++ b/tensorflow_text/core/kernels/phrase_tokenizer_kernel_template.h @@ -126,7 +126,7 @@ absl::Status PhraseTokenizeOp::Invoke(InvokeContext* context) { // lightweight, memory-mapped wrapper on `phrase_model` tensor, and thus // Create() is very cheap. auto phrase_tokenizer = ::tensorflow::text::PhraseTokenizer::Create( - phrase_model->template Data().data()); + phrase_model->template Data().data()); SH_RETURN_IF_ERROR(phrase_tokenizer.status()); std::vector subwords; @@ -159,13 +159,13 @@ absl::Status PhraseTokenizeOp::Invoke(InvokeContext* context) { kOutputIds, Shape({static_cast( subword_ids.size())}))); /* same shape as `output_subwords` */ - auto output_ids_vec = output_ids->template As(); + auto output_ids_vec = output_ids->template As(); SH_ASSIGN_OR_RETURN( auto output_row_splits, context->GetOutput(kOutputRowSplits, Shape({static_cast(row_splits.size())}))); - auto output_row_splits_vec = output_row_splits->template As(); + auto output_row_splits_vec = output_row_splits->template As(); for (int i = 0; i < subwords.size(); ++i) { output_subwords_vec(i) = subwords[i]; @@ -299,14 +299,14 @@ absl::Status PhraseDetokenizeOp::Invoke(InvokeContext* context) { SH_ASSIGN_OR_RETURN(const auto input_row_splits, context->GetInput(kInputRowSplits)); - const auto& row_splits_vec = input_row_splits->template As(); + const auto& row_splits_vec = input_row_splits->template As(); SH_ASSIGN_OR_RETURN(const auto phrase_model, context->GetInput(kPhraseModel)); // OK to create on every call because PhraseTokenizer is a // lightweight, memory-mapped wrapper on `phrase_model` tensor, and thus // Create() is very cheap. auto phrase_tokenizer = ::tensorflow::text::PhraseTokenizer::Create( - phrase_model->template Data().data()); + phrase_model->template Data().data()); SH_RETURN_IF_ERROR(phrase_tokenizer.status()); std::vector sentences; diff --git a/tensorflow_text/core/kernels/regex_split_kernels.cc b/tensorflow_text/core/kernels/regex_split_kernels.cc index fd79a63d7..dcd832774 100644 --- a/tensorflow_text/core/kernels/regex_split_kernels.cc +++ b/tensorflow_text/core/kernels/regex_split_kernels.cc @@ -42,7 +42,7 @@ class RegexSplitOp : public tensorflow::OpKernel { errors::InvalidArgument( "Pattern must be scalar, but received ", delim_regex_pattern_tensor->shape().DebugString())); - const string delim_regex_pattern = + const std::string delim_regex_pattern = delim_regex_pattern_tensor->flat()(0); delim_re = CachedDelimRE2(delim_regex_pattern); OP_REQUIRES( @@ -59,7 +59,7 @@ class RegexSplitOp : public tensorflow::OpKernel { errors::InvalidArgument( "Pattern must be scalar, but received ", keep_delim_regex_pattern_tensor->shape().DebugString())); - const string keep_delim_regex_pattern = + const std::string keep_delim_regex_pattern = keep_delim_regex_pattern_tensor->flat()(0); keep_delim_re = CachedKeepDelimRE2(keep_delim_regex_pattern); OP_REQUIRES( @@ -73,10 +73,10 @@ class RegexSplitOp : public tensorflow::OpKernel { OP_REQUIRES_OK(ctx, ctx->input("input", &input_tensor)); const auto& input_flat = input_tensor->flat(); - std::vector begin_offsets; - std::vector end_offsets; + std::vector begin_offsets; + std::vector end_offsets; std::vector tokens; - std::vector row_splits; + std::vector row_splits; row_splits.push_back(0); for (size_t i = 0; i < input_flat.size(); ++i) { @@ -88,13 +88,13 @@ class RegexSplitOp : public tensorflow::OpKernel { // Emit the flat Tensors needed to construct RaggedTensors for tokens, // start, end offsets. - std::vector tokens_shape; + std::vector tokens_shape; tokens_shape.push_back(tokens.size()); - std::vector offsets_shape; + std::vector offsets_shape; offsets_shape.push_back(begin_offsets.size()); - std::vector row_splits_shape; + std::vector row_splits_shape; row_splits_shape.push_back(row_splits.size()); Tensor* output_tokens_tensor = nullptr; @@ -107,19 +107,19 @@ class RegexSplitOp : public tensorflow::OpKernel { OP_REQUIRES_OK( ctx, ctx->allocate_output("begin_offsets", TensorShape(offsets_shape), &output_begin_offsets_tensor)); - auto output_begin_offsets = output_begin_offsets_tensor->flat(); + auto output_begin_offsets = output_begin_offsets_tensor->flat(); Tensor* output_end_offsets_tensor = nullptr; OP_REQUIRES_OK( ctx, ctx->allocate_output("end_offsets", TensorShape(offsets_shape), &output_end_offsets_tensor)); - auto output_end_offsets = output_end_offsets_tensor->flat(); + auto output_end_offsets = output_end_offsets_tensor->flat(); Tensor* output_row_splits_tensor = nullptr; OP_REQUIRES_OK( ctx, ctx->allocate_output("row_splits", TensorShape(row_splits_shape), &output_row_splits_tensor)); - auto output_row_splits = output_row_splits_tensor->flat(); + auto output_row_splits = output_row_splits_tensor->flat(); // Copy outputs to Tensors. for (size_t i = 0; i < tokens.size(); ++i) { @@ -141,7 +141,7 @@ class RegexSplitOp : public tensorflow::OpKernel { } private: - std::shared_ptr CachedDelimRE2(const string& pattern) { + std::shared_ptr CachedDelimRE2(const std::string& pattern) { { tf_shared_lock l(delim_mu_); if (delim_re_ != nullptr && delim_re_->pattern() == pattern) { @@ -159,7 +159,7 @@ class RegexSplitOp : public tensorflow::OpKernel { } } - std::shared_ptr CachedKeepDelimRE2(const string& pattern) { + std::shared_ptr CachedKeepDelimRE2(const std::string& pattern) { { tf_shared_lock l(keep_delim_mu_); if (keep_delim_re_ != nullptr && keep_delim_re_->pattern() == pattern) { diff --git a/tensorflow_text/core/kernels/rouge_l_kernel.cc b/tensorflow_text/core/kernels/rouge_l_kernel.cc index 50a730f47..919e755d6 100644 --- a/tensorflow_text/core/kernels/rouge_l_kernel.cc +++ b/tensorflow_text/core/kernels/rouge_l_kernel.cc @@ -113,12 +113,9 @@ class RougeLOp : public OpKernel { SPLITS_TYPE lhyp = hyp_splits_flat(i) - hyp_splits_flat(i-1); SPLITS_TYPE lref = ref_splits_flat(i) - ref_splits_flat(i-1); // Length of longest common substring. - int32 llcs = LongestCommonSubsequenceLength(hyp_splits_flat(i-1), - hyp_splits_flat(i), - hyp_tensor_flat, - ref_splits_flat(i-1), - ref_splits_flat(i), - ref_tensor_flat); + int32_t llcs = LongestCommonSubsequenceLength( + hyp_splits_flat(i - 1), hyp_splits_flat(i), hyp_tensor_flat, + ref_splits_flat(i - 1), ref_splits_flat(i), ref_tensor_flat); auto measures = ComputeMeasures(lhyp, lref, llcs, alpha); f_measures_flat(i - 1) = std::get<0>(measures); p_measures_flat(i - 1) = std::get<1>(measures); @@ -129,19 +126,18 @@ class RougeLOp : public OpKernel { private: // By using LCS, the ROUGE-L algorithm does not require consecutive matches // but rather credits the order of N-grams. - int32 LongestCommonSubsequenceLength( - const SPLITS_TYPE hyp_i, - const SPLITS_TYPE hyp_j, - const ConstFlatValues& hyp, - const SPLITS_TYPE ref_i, - const SPLITS_TYPE ref_j, - const ConstFlatValues& ref) { + int32_t LongestCommonSubsequenceLength(const SPLITS_TYPE hyp_i, + const SPLITS_TYPE hyp_j, + const ConstFlatValues& hyp, + const SPLITS_TYPE ref_i, + const SPLITS_TYPE ref_j, + const ConstFlatValues& ref) { SPLITS_TYPE lhyp = hyp_j - hyp_i; SPLITS_TYPE lref = ref_j - ref_i; // Create a scratch matrix to keep track of the LCS seen so far using DP. // http://www.algorithmist.com/index.php/Longest_Common_Subsequence Tensor scratch(DT_INT32, {lhyp + 2, lref + 2}); - auto scratch2d = scratch.matrix(); + auto scratch2d = scratch.matrix(); for (SPLITS_TYPE x = hyp_i; x <= hyp_j + 1; x++) { for (SPLITS_TYPE y = ref_i; y <= ref_j + 1; y++) { SPLITS_TYPE a = x - hyp_i; @@ -167,7 +163,7 @@ class RougeLOp : public OpKernel { std::tuple ComputeMeasures(const SPLITS_TYPE lhyp_int, const SPLITS_TYPE lref_int, - const int32 llcs_int, + const int32_t llcs_int, const float alpha) { const float lhyp = static_cast(lhyp_int); const float lref = static_cast(lref_int); diff --git a/tensorflow_text/core/kernels/round_robin_trimmer_kernel_template.h b/tensorflow_text/core/kernels/round_robin_trimmer_kernel_template.h index 51f17da43..b38b4eedb 100644 --- a/tensorflow_text/core/kernels/round_robin_trimmer_kernel_template.h +++ b/tensorflow_text/core/kernels/round_robin_trimmer_kernel_template.h @@ -147,7 +147,7 @@ template absl::Status RoundRobinTrimOp::Invoke(InvokeContext* context) { // Inputs SH_ASSIGN_OR_RETURN(const auto msl, context->GetInput(kMaxSeqLength)); - const int max_sequence_length = msl->template AsScalar(); + const int max_sequence_length = msl->template AsScalar(); std::vector> list_of_values(number_of_segments_); std::vector> list_of_splits(number_of_segments_); @@ -295,7 +295,7 @@ absl::Status RoundRobinGenerateMasksOp::Invoke( InvokeContext* context) { // Inputs SH_ASSIGN_OR_RETURN(const auto msl, context->GetInput(kMaxSeqLength)); - const int max_sequence_length = msl->template AsScalar(); + const int max_sequence_length = msl->template AsScalar(); std::vector> list_of_splits(number_of_segments_); for (int i = 0; i < number_of_segments_; ++i) { diff --git a/tensorflow_text/core/kernels/sentence_breaking_kernels.cc b/tensorflow_text/core/kernels/sentence_breaking_kernels.cc index 0f4c34c82..55a1b0c38 100644 --- a/tensorflow_text/core/kernels/sentence_breaking_kernels.cc +++ b/tensorflow_text/core/kernels/sentence_breaking_kernels.cc @@ -47,7 +47,7 @@ class WrappedConverter { } } - void init(const string& name) { + void init(const std::string& name) { if (converter_ && name == name_) { // Note: this reset is not typically needed, but if not done, then in some // cases the cached converter will maintain state of input endianness @@ -75,7 +75,7 @@ class WrappedConverter { } UConverter* converter_ = nullptr; - string name_; + std::string name_; }; struct ErrorOptions { @@ -88,7 +88,7 @@ struct ErrorOptions { absl::Status GetErrorOptions(OpKernelConstruction* context, ErrorOptions* out) { *out = ErrorOptions(); - string error_policy; + std::string error_policy; TF_RETURN_IF_ERROR(context->GetAttr("errors", &error_policy)); if (error_policy == "replace") { @@ -102,7 +102,7 @@ absl::Status GetErrorOptions(OpKernelConstruction* context, ErrorOptions* out) { "errors policy must be one of 'strict', 'replace', or 'ignore'"); } - int32 replacement_char; + int32_t replacement_char; TF_RETURN_IF_ERROR(context->GetAttr("replacement_char", &replacement_char)); if (replacement_char >= UCHAR_MIN_VALUE && @@ -157,11 +157,11 @@ class SentenceFragmentsOp : public OpKernel { name##_tensor->shape().DebugString()))); \ const auto& name = name##_tensor->vec(); - DECLARE_AND_VALIDATE_INPUT_VECTOR(row_lengths, int64); - DECLARE_AND_VALIDATE_INPUT_VECTOR(token_start, int64); - DECLARE_AND_VALIDATE_INPUT_VECTOR(token_end, int64); + DECLARE_AND_VALIDATE_INPUT_VECTOR(row_lengths, int64_t); + DECLARE_AND_VALIDATE_INPUT_VECTOR(token_start, int64_t); + DECLARE_AND_VALIDATE_INPUT_VECTOR(token_end, int64_t); DECLARE_AND_VALIDATE_INPUT_VECTOR(token_word, tstring); - DECLARE_AND_VALIDATE_INPUT_VECTOR(token_properties, int64); + DECLARE_AND_VALIDATE_INPUT_VECTOR(token_properties, int64_t); #undef DECLARE_AND_VALIDATE_INPUT_TENSOR @@ -216,10 +216,10 @@ class SentenceFragmentsOp : public OpKernel { fragments.push_back(std::move(frags)); } - std::vector fragment_shape; + std::vector fragment_shape; fragment_shape.push_back(num_fragments); - std::vector doc_batch_shape; + std::vector doc_batch_shape; doc_batch_shape.push_back(fragments.size()); #define DECLARE_OUTPUT_TENSOR(name, out_shape) \ @@ -256,7 +256,7 @@ class SentenceFragmentsOp : public OpKernel { } private: - string input_encoding_; + std::string input_encoding_; ErrorOptions error_options_; }; diff --git a/tensorflow_text/core/kernels/sentence_fragmenter.cc b/tensorflow_text/core/kernels/sentence_fragmenter.cc index c336b5cfa..a571c72f4 100644 --- a/tensorflow_text/core/kernels/sentence_fragmenter.cc +++ b/tensorflow_text/core/kernels/sentence_fragmenter.cc @@ -30,7 +30,7 @@ void SetFragmentProperty(SentenceFragment::Property property, } // Returns true iff a token has any of the given properties. -bool TokenHasProperty(uint32 properties, const Token &token) { +bool TokenHasProperty(uint32_t properties, const Token& token) { return token.text_properties() & properties; } diff --git a/tensorflow_text/core/kernels/sentence_fragmenter.h b/tensorflow_text/core/kernels/sentence_fragmenter.h index 25f1038e1..6bd7d6d69 100644 --- a/tensorflow_text/core/kernels/sentence_fragmenter.h +++ b/tensorflow_text/core/kernels/sentence_fragmenter.h @@ -106,8 +106,8 @@ class Token { HYPERLINK = 0x200, }; - Token(const tstring &word, uint32 start, uint32 end, BreakLevel break_level, - TextProperty text_properties) + Token(const tstring& word, uint32_t start, uint32_t end, + BreakLevel break_level, TextProperty text_properties) : word_(word), start_(start), end_(end), @@ -115,15 +115,15 @@ class Token { text_properties_(text_properties) {} const tstring &word() const { return word_; } - const uint32 start() const { return start_; } - const uint32 end() const { return end_; } + const uint32_t start() const { return start_; } + const uint32_t end() const { return end_; } const BreakLevel break_level() const { return break_level_; } const TextProperty text_properties() const { return text_properties_; } private: const tstring &word_; - uint32 start_; - uint32 end_; + uint32_t start_; + uint32_t end_; BreakLevel break_level_; TextProperty text_properties_; }; @@ -133,7 +133,7 @@ class Document { // Does NOT take ownership of 'tokens'. Document(std::vector *tokens) : tokens_(tokens) {} - void AddToken(const tstring &word, uint32 start, uint32 end, + void AddToken(const tstring& word, uint32_t start, uint32_t end, Token::BreakLevel break_level, Token::TextProperty text_properties) { tokens_->emplace_back(word, start, end, break_level, text_properties); @@ -157,7 +157,7 @@ struct SentenceFragment { HAS_SENTENTIAL_CLOSE_PAREN = 0x0008, // e.g.: (Mushrooms are fungi!) }; // A mask of the above listed properties. - uint32 properties = 0; + uint32_t properties = 0; int terminal_punc_token = -1; }; diff --git a/tensorflow_text/core/kernels/sentence_fragmenter_v2.h b/tensorflow_text/core/kernels/sentence_fragmenter_v2.h index 6c06867eb..2d369b5a9 100644 --- a/tensorflow_text/core/kernels/sentence_fragmenter_v2.h +++ b/tensorflow_text/core/kernels/sentence_fragmenter_v2.h @@ -140,7 +140,7 @@ struct SentenceFragment { HAS_SENTENTIAL_CLOSE_PAREN = 0x0008, // e.g.: (Mushrooms are fungi!) }; // A mask of the above listed properties. - uint32 properties = 0; + uint32_t properties = 0; int terminal_punc_token = -1; }; diff --git a/tensorflow_text/core/kernels/sentence_fragmenter_v2_kernel_template.h b/tensorflow_text/core/kernels/sentence_fragmenter_v2_kernel_template.h index ea7be5862..1cd7ba2e3 100644 --- a/tensorflow_text/core/kernels/sentence_fragmenter_v2_kernel_template.h +++ b/tensorflow_text/core/kernels/sentence_fragmenter_v2_kernel_template.h @@ -120,11 +120,11 @@ absl::Status SentenceFragmenterV2Op::Invoke(InvokeContext* context) { const auto document = input_values->template As(); // Outputs - std::vector fragment_start; - std::vector fragment_end; - std::vector fragment_properties; - std::vector terminal_punc_token; - std::vector output_row_lengths; + std::vector fragment_start; + std::vector fragment_end; + std::vector fragment_properties; + std::vector terminal_punc_token; + std::vector output_row_lengths; // Iterate through all the documents and find fragments. for (int i = 0; i < document.Dim(0); ++i) { diff --git a/tensorflow_text/core/kernels/sentence_fragmenter_v2_test.cc b/tensorflow_text/core/kernels/sentence_fragmenter_v2_test.cc index 87cd49265..e7a9ad030 100644 --- a/tensorflow_text/core/kernels/sentence_fragmenter_v2_test.cc +++ b/tensorflow_text/core/kernels/sentence_fragmenter_v2_test.cc @@ -736,7 +736,7 @@ TEST_F(IsEmoticonTest, IsEmoticon) { TEST(SentenceFragmenterTest, Basic) { // 1 // 012345678901234 - string test_input = "Hello. Foo bar!"; + std::string test_input = "Hello. Foo bar!"; SentenceFragmenterV2 fragmenter(test_input); std::vector fragments; EXPECT_TRUE(fragmenter.FindFragments(&fragments).ok()); @@ -749,7 +749,7 @@ TEST(SentenceFragmenterTest, Basic) { TEST(SentenceFragmenterTest, BasicEllipsis) { // 1 // 012345678901234 - string test_input = "Hello...foo bar"; + std::string test_input = "Hello...foo bar"; SentenceFragmenterV2 fragmenter(test_input); std::vector fragments; EXPECT_TRUE(fragmenter.FindFragments(&fragments).ok()); @@ -763,7 +763,7 @@ TEST(SentenceFragmenterTest, BasicEllipsis) { TEST(SentenceFragmenterTest, Parentheses) { // 1 2 // 012345678901234567890123456789 - string test_input = "Hello (who are you...) foo bar"; + std::string test_input = "Hello (who are you...) foo bar"; SentenceFragmenterV2 fragmenter(test_input); std::vector fragments; EXPECT_TRUE(fragmenter.FindFragments(&fragments).ok()); @@ -776,7 +776,7 @@ TEST(SentenceFragmenterTest, Parentheses) { TEST(SentenceFragmenterTest, MidFragmentParentheses) { // 1 2 // 012345678901234567890123456789 - string test_input = "Hello (who are you) world? Foo bar"; + std::string test_input = "Hello (who are you) world? Foo bar"; SentenceFragmenterV2 fragmenter(test_input); std::vector fragments; EXPECT_TRUE(fragmenter.FindFragments(&fragments).ok()); @@ -789,7 +789,7 @@ TEST(SentenceFragmenterTest, MidFragmentParentheses) { TEST(SentenceFragmenterTest, PunctuationAfterParentheses) { // 1 2 // 01234567890123456789012345678 - string test_input = "Hello (who are you)? Foo bar!"; + std::string test_input = "Hello (who are you)? Foo bar!"; SentenceFragmenterV2 fragmenter(test_input); std::vector fragments; EXPECT_TRUE(fragmenter.FindFragments(&fragments).ok()); @@ -802,7 +802,7 @@ TEST(SentenceFragmenterTest, PunctuationAfterParentheses) { TEST(SentenceFragmenterTest, ManyFinalPunctuations) { // 1 2 // 0123456789012345678901234 - string test_input = "Hello!!!!! Who are you??"; + std::string test_input = "Hello!!!!! Who are you??"; SentenceFragmenterV2 fragmenter(test_input); std::vector fragments; EXPECT_TRUE(fragmenter.FindFragments(&fragments).ok()); @@ -815,7 +815,7 @@ TEST(SentenceFragmenterTest, ManyFinalPunctuations) { TEST(SentenceFragmenterTest, NewLine) { // 1 2 3 // 012345678901234567890 1 23456 7 89012 3 45678 - string test_input = "Who let the dogs out?\r\nWho?\r\nWho?\r\nWho?"; + std::string test_input = "Who let the dogs out?\r\nWho?\r\nWho?\r\nWho?"; SentenceFragmenterV2 fragmenter(test_input); std::vector fragments; EXPECT_TRUE(fragmenter.FindFragments(&fragments).ok()); @@ -832,7 +832,7 @@ TEST(SentenceFragmenterTest, NewLine) { TEST(SentenceFragmenterTest, WhiteSpaceInPunctuation) { // 1 2 // 0123456789012345678901234 - string test_input = "Hello?? !!! Who are you??"; + std::string test_input = "Hello?? !!! Who are you??"; SentenceFragmenterV2 fragmenter(test_input); std::vector fragments; EXPECT_TRUE(fragmenter.FindFragments(&fragments).ok()); @@ -850,7 +850,7 @@ TEST(FragmentBoundaryMatchTest, NoStateChange) { FragmentBoundaryMatch f; // || // 012345678901234 - string test_input = "Hello...foo bar"; + std::string test_input = "Hello...foo bar"; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_FALSE(f.GotTerminalPunc()); @@ -864,7 +864,7 @@ TEST(FragmentBoundaryMatchTest, BasicEllipsis) { FragmentBoundaryMatch f; // | | // 0123456789 - string test_input = "...foo bar"; + std::string test_input = "...foo bar"; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_TRUE(f.GotTerminalPunc()); @@ -878,7 +878,7 @@ TEST(FragmentBoundaryMatchTest, BasicPeriod) { FragmentBoundaryMatch f; // || // 0123456789 - string test_input = ". Foo bar"; + std::string test_input = ". Foo bar"; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_TRUE(f.GotTerminalPunc()); @@ -892,7 +892,7 @@ TEST(FragmentBoundaryMatchTest, BasicAcronym) { FragmentBoundaryMatch f; // | | // 0123456789 - string test_input = "A.B. xyz"; + std::string test_input = "A.B. xyz"; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_TRUE(f.GotTerminalPunc()); @@ -906,7 +906,7 @@ TEST(FragmentBoundaryMatchTest, LongerAcronym) { FragmentBoundaryMatch f; // | | // 0123456789 - string test_input = "I.B.M. yo"; + std::string test_input = "I.B.M. yo"; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_TRUE(f.GotTerminalPunc()); @@ -920,7 +920,7 @@ TEST(FragmentBoundaryMatchTest, Emoticon) { FragmentBoundaryMatch f; // | | // 0123456789012 - string test_input = ">:-( hello..."; + std::string test_input = ">:-( hello..."; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_TRUE(f.GotTerminalPunc()); @@ -934,7 +934,7 @@ TEST(FragmentBoundaryMatchTest, ParensWithEllipsis) { FragmentBoundaryMatch f; // || // 0123456789012345 - string test_input = ".foo...) foo bar"; + std::string test_input = ".foo...) foo bar"; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_TRUE(f.GotTerminalPunc()); @@ -948,7 +948,7 @@ TEST(FragmentBoundaryMatchTest, ClosingParenWithEllipsis) { FragmentBoundaryMatch f; // | | // 012345678901 - string test_input = "...) foo bar"; + std::string test_input = "...) foo bar"; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_TRUE(f.GotTerminalPunc()); @@ -962,7 +962,7 @@ TEST(FragmentBoundaryMatchTest, BeginAndEndParenWithEllipsis) { FragmentBoundaryMatch f; // || // 0123456789012 - string test_input = "(...) foo bar"; + std::string test_input = "(...) foo bar"; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_FALSE(f.GotTerminalPunc()); @@ -986,7 +986,7 @@ TEST(FragmentBoundaryMatchTest, AcronymInSentence) { FragmentBoundaryMatch f; // | | // 0123456789012 - string test_input = "U.S. don't be surprised."; + std::string test_input = "U.S. don't be surprised."; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_TRUE(f.GotTerminalPunc()); @@ -1000,7 +1000,7 @@ TEST(FragmentBoundaryMatchTest, HelloWithEllipsis) { FragmentBoundaryMatch f; // || // 01234567890 - string test_input = "o...foo bar"; + std::string test_input = "o...foo bar"; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_FALSE(f.GotTerminalPunc()); @@ -1024,7 +1024,7 @@ TEST(FragmentBoundaryMatchTest, ThreeStatesWithClosigParen) { FragmentBoundaryMatch f; // || // 0123456789012 - string test_input = "w...) foo bar"; + std::string test_input = "w...) foo bar"; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_FALSE(f.GotTerminalPunc()); @@ -1068,7 +1068,7 @@ TEST(FragmentBoundaryMatchTest, NoTransition) { FragmentBoundaryMatch f; // | | // 0123456789012 - string test_input = "...foo bar"; + std::string test_input = "...foo bar"; int index = 0; EXPECT_TRUE(f.Advance(index, test_input)); EXPECT_TRUE(f.GotTerminalPunc());