implement copy task

Author: ShigekiKarita

Makefile

@@ -1,8 +1,30 @@
+export CXX_FLAGS := $(CXXFLAGS) -std=c++17 -g3 -O0 -pthread -D_GLIBCXX_USE_CXX11_ABI=0 -Wall -Wextra -Wno-unused-function -D_LIBCPP_DEBUG # -D_GLIBCXX_DEBUG
+# -ftrapv
+
+export INCPATH := $(shell python -c "import torch.utils.cpp_extension as C; print('-isystem' + str.join(' -isystem', C.include_paths()))")
+
+export LIBPATH := $(shell python -c "import torch.utils.cpp_extension as C; print(C.include_paths()[0] + '/../')")
+
+export USE_CUDA := $(shell python -c "import torch; print(torch.cuda.is_available())")
+
+ifeq ($(USE_CUDA),True)
+export TORCH_LIBS=-ltorch -lcaffe2 -lcaffe2_gpu -lc10 -lc10_cuda -lcuda -lnvrtc -lnvToolsExt # -lnccl -lmkldnn -lmkl_rt
+else
+export TORCH_LIBS=-ltorch -lcaffe2 -lc10
+endif
+
+
 .PHONY: test clean install-conda install-latest
 
 test:
 	$(MAKE) --directory test
 
+example-mnist:
+	$(MAKE) --directory example/mnist
+
+example-copy:
+	$(MAKE) --directory example/copy
+
 clean:
 	find . -name "*.o" -exec rm -v {} \;
 	find . -name "*.out" -exec rm -v {} \;

example/copy/Makefile

@@ -0,0 +1,7 @@
+.PHONY: run
+
+run: main.out
+	./main.out
+
+main.out: main.cpp
+	$(CXX) $(CXX_FLAGS) $^ -o $@  -Wl,-rpath,$(LIBPATH) -Wl,-rpath,$(CONDA_PREFIX)/lib $(TORCH_LIBS) $(KALDI_FLAGS) -L$(CONDA_PREFIX)/lib -L$(LIBPATH) $(INCPATH) -I../../include

example/copy/main.cpp

@@ -1,16 +1,41 @@
+/**
+   toy task introduced in https://github.com/harvardnlp/annotated-transformer
+   TODO: cuda backend, greedy/beam decoding
+*/
 #include <torch/torch.h>
+#include <thxx/net.hpp>
 
 constexpr std::int64_t idim = 11;
 constexpr std::int64_t odim = 11;
 
-auto gen_data(std::int64_t batch_size) {
-    auto x = torch::randint(0, idim - 1, {batch_size, 10});
-    auto xlen = torch::empty({batch_size});
+auto gen_data(std::int64_t batch_size = 128) {
+    auto x = torch::randint(1, idim - 1, {batch_size, 10}).to(at::kLong);
+    std::vector<std::int64_t> xlen(batch_size, 10);
     return std::make_tuple(x, xlen);
 }
 
 int main() {
-    for (int i = 0; i < 100; ++i) {
+    thxx::net::transformer::Config config;
+    config.heads = 2;
+    config.d_model = 16;
+    config.d_ff = 16;
+    config.elayers = 1;
+    config.dlayers = 1;
+
+    using InputLayer = thxx::net::transformer::PositonalEmbedding;
+    thxx::net::Transformer<InputLayer> model(idim, odim, config);
+    torch::optim::Adam optimizer(model.parameters(), 0.01);
+
+    for (int i = 1; i <= 1000; ++i) {
         auto [x, xlen] = gen_data();
+        optimizer.zero_grad();
+        auto [loss, acc] = model.forward(x, xlen, x, xlen);
+        loss.backward();
+        optimizer.step();
+        if (i % 100 == 0) {
+            std::cout << "step: " << i
+                      << ", loss: " << loss.template item<double>()
+                      << ", acc: " << acc << std::endl;
+        }
     }
 }

include/thxx/net.hpp

@@ -528,7 +528,8 @@ namespace thxx {
                 auto target = tgt_out.view({-1});
                 auto loss = label_smoothing_kl_div(pred.view({target.size(0), -1}), target,
                                                    this->config.label_smoothing, this->ignore_index);
-                return loss;
+                auto acc = accuracy(pred, tgt_out, this->ignore_index);
+                return std::make_tuple(loss, acc);
             }
         };
     } // namespace net

test/Makefile

@@ -1,20 +1,3 @@
-# INSTALL: conda install pytorch-cpu=1.0.0 -c pytorch
-
-CXX_FLAGS := $(CXXFLAGS) -std=c++17 -g3 -O0 -pthread -D_GLIBCXX_USE_CXX11_ABI=0 -Wall -Wextra -Wno-unused-function -D_LIBCPP_DEBUG # -D_GLIBCXX_DEBUG
-# -ftrapv
-
-INCPATH := $(shell python -c "import torch.utils.cpp_extension as C; print('-isystem' + str.join(' -isystem', C.include_paths()))")
-
-LIBPATH := $(shell python -c "import torch.utils.cpp_extension as C; print(C.include_paths()[0] + '/../')")
-
-USE_CUDA := $(shell python -c "import torch; print(torch.cuda.is_available())")
-
-ifeq ($(USE_CUDA),True)
-	TORCH_LIBS=-ltorch -lcaffe2 -lcaffe2_gpu -lc10 -lc10_cuda -lcuda -lnvrtc -lnvToolsExt # -lnccl -lmkldnn -lmkl_rt 
-else
-	TORCH_LIBS=-ltorch -lcaffe2 -lc10
-endif
-
 .PHONY: all
 
 all: test.out

test/test_net.cpp

@@ -161,13 +161,13 @@ TEST_CASE( "transformer", "[net]" ) {
     }
     {
         Transformer<T::Conv2dSubsampling> model(n_input, n_output, conf);
-        auto loss = model.forward(x, xlen, t, ylen);
+        auto [loss, acc] = model.forward(x, xlen, t, ylen);
         loss.backward();
         CHECK_THAT( model, testing::HasGrad(true) );
     }
     {
         Transformer<T::PositonalEmbedding> model(n_input, n_output, conf);
-        auto loss = model.forward(t, ylen, t, ylen);
+        auto [loss, acc] = model.forward(t, ylen, t, ylen);
         loss.backward();
         CHECK_THAT( model, testing::HasGrad(true) );
     }