v0.1.0 (#60)

README.md

@@ -21,15 +21,23 @@ ARK is under active development and a part of its features will be added in a fu
 
 ### ARK v0.1 (Latest Release)
 
-TBU
+* The default tile-based operator scheduler
+* A simple software communication stack
+* Transformer inference examples
 
-### ARK v0.2 (TBU)
+### ARK v0.2 (TBU, Sep. 2023)
 
-TBU
+* A simple operator scheduler for debugging
+* Extended communication interfaces
+* Support more operators
+* More inference examples
 
-### ARK v0.3 (TBU)
+### ARK v0.3 (TBU, Nov. 2023)
 
-TBU
+* Full support for the operator profiler
+* High-performance collective communication
+* Support more operators
+* More inference & training examples
 
 ## Contributing
 

ark/init.cc → ark/ark.cc

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+#include <sstream>
 #include <string>
 #include <vector>
 
@@ -13,10 +14,13 @@
 
 namespace ark {
 
-/// Initialize the ARK runtime.
-///
-/// This function should be called by the user before any other functions are
-/// called. It is safe to call this function multiple times.
+std::string version()
+{
+    std::stringstream ss;
+    ss << ARK_MAJOR << "." << ARK_MINOR << "." << ARK_PATCH;
+    return ss.str();
+}
+
 void init()
 {
     LOG(DEBUG, "init ark");

ark/include/ark.h

@@ -4,24 +4,30 @@
 #ifndef ARK_H
 #define ARK_H
 
-#include <array>
-#include <cassert>
-#include <iostream>
-#include <list>
-#include <map>
 #include <memory>
-#include <set>
 #include <string>
 #include <vector>
 
+#define ARK_MAJOR 0
+#define ARK_MINOR 1
+#define ARK_PATCH 0
+#define ARK_VERSION (ARK_MAJOR * 10000 + ARK_MINOR * 100 + ARK_PATCH)
+
 namespace ark {
 
+/// Return a version string.
+std::string version();
+
 // set random seed
 void srand(int seed = -1);
+
+// get random number
 int rand();
 
-// Init an ark program. Call this function to clean up the shared memory
-// directory
+/// Initialize the ARK runtime.
+///
+/// This function should be called by the user before any other functions are
+/// called. It is safe to call this function multiple times.
 void init();
 
 // Data type for dimension.

ark/include/ark_utils.h

@@ -5,17 +5,8 @@
 #define ARK_UTILS_H
 
 #include "ark.h"
-#include <array>
-#include <cassert>
-#include <cstring>
 #include <functional>
-#include <iostream>
-#include <list>
-#include <map>
 #include <memory>
-#include <set>
-#include <string>
-#include <utility>
 #include <vector>
 
 namespace ark {

ark/model.h

@@ -6,6 +6,9 @@
 
 #include "include/ark.h"
 #include "ops/ops_common.h"
+#include <list>
+#include <map>
+#include <set>
 
 namespace ark {
 

ark/ops/ops_all_gather.cc

@@ -5,14 +5,13 @@
 #include "math.h"
 #include "model.h"
 #include "ops_common.h"
-
-using namespace std;
+#include <cassert>
 
 namespace ark {
 
 std::vector<Tensor *> Model::all_gather(Tensor *input, int gpu_id, int gpu_num,
                                         std::vector<Tensor *> output,
-                                        const string &)
+                                        const std::string &)
 {
     assert(input != nullptr);
     LOG(DEBUG, "all_gather ", input->shape, " ", gpu_id, " ", gpu_num);
@@ -24,7 +23,7 @@ std::vector<Tensor *> Model::all_gather(Tensor *input, int gpu_id, int gpu_num,
     LOG(DEBUG, "all gather output size: ", output.size());
 
     int base = this->impl->next_eid;
-    vector<Tensor *> recv_dep_tensors;
+    std::vector<Tensor *> recv_dep_tensors;
     for (int gpu_dst = 0; gpu_dst < gpu_num; gpu_dst++) {
         if (gpu_dst == gpu_id)
             continue;

ark/ops/ops_all_reduce.cc

@@ -5,13 +5,12 @@
 #include "math.h"
 #include "model.h"
 #include "ops_common.h"
-
-using namespace std;
+#include <cassert>
 
 namespace ark {
 
 Tensor *Model::all_reduce(Tensor *input, int gpu_id, int gpu_num,
-                          Tensor *output, const string &)
+                          Tensor *output, const std::string &)
 {
     assert(input != nullptr);
     if (output != nullptr) {

ark/ops/ops_common.h

@@ -5,6 +5,7 @@
 #define ARK_OPS_COMMON_H_
 
 #include "include/ark.h"
+#include <map>
 #include <ostream>
 #include <vector>
 

ark/ops/ops_gelu.cc

@@ -4,15 +4,14 @@
 #include "logging.h"
 #include "model.h"
 #include "tensor.h"
-
-using namespace std;
+#include <cassert>
 
 namespace ark {
 
 extern const OpConfigMap ActivationConfigMap;
 
 GeluOp::GeluOp(OpPrecType prec_type, Tensor *input, Tensor *output,
-               const string &name)
+               const std::string &name)
     : Op{OP_GELU, prec_type, {input}, {output}, {}, name, &ActivationConfigMap,
          -1,      true}
 {
@@ -44,7 +43,7 @@ std::string GeluOp::function_name(const OpConfig &cfg) const
                                           }});
 }
 
-Tensor *Model::gelu(Tensor *input, Tensor *output, const string &name)
+Tensor *Model::gelu(Tensor *input, Tensor *output, const std::string &name)
 {
     assert(input != nullptr);
     OpPrecType pt;

ark/ops/ops_identity.cc

@@ -3,23 +3,22 @@
 
 #include "logging.h"
 #include "model.h"
-
-using namespace std;
+#include <cassert>
 
 namespace ark {
 
 // Returns an identical tensor of `input` with execution dependencies `deps`.
-Tensor *Model::identity(Tensor *input, const vector<Tensor *> &deps,
-                        const string &name)
+Tensor *Model::identity(Tensor *input, const std::vector<Tensor *> &deps,
+                        const std::string &name)
 {
     assert(input != nullptr);
     LOG(DEBUG, "identity ", input->shape);
-    set<Tensor *> dep_set;
+    std::set<Tensor *> dep_set;
     dep_set.emplace(input);
     for (auto &dep : deps) {
         dep_set.emplace(dep);
     }
-    vector<Tensor *> dep_vec;
+    std::vector<Tensor *> dep_vec;
     for (auto &dep : dep_set) {
         dep_vec.emplace_back(dep);
     }

ark/ops/ops_im2col.cc

@@ -4,8 +4,7 @@
 #include "logging.h"
 #include "model.h"
 #include "tensor.h"
-
-using namespace std;
+#include <cassert>
 
 namespace ark {
 
@@ -14,7 +13,8 @@ extern const OpConfigMap Im2colConfigMap;
 Im2colOp::Im2colOp(OpPrecType prec_type, Tensor *input, Tensor *output,
                    int kernel_height, int kernel_width, int stride_height,
                    int stride_width, int pad_height, int pad_width,
-                   int dilation_height, int dilation_width, const string &name)
+                   int dilation_height, int dilation_width,
+                   const std::string &name)
     : Op{OP_IM2COL,
          prec_type,
          {input},
@@ -83,7 +83,7 @@ std::string Im2colOp::function_name(const OpConfig &cfg) const
 Tensor *Model::im2col(Tensor *input, int kernel_height, int kernel_width,
                       int stride_height, int stride_width, int pad_height,
                       int pad_width, int dilation_height, int dilation_width,
-                      Tensor *output, const string &name)
+                      Tensor *output, const std::string &name)
 {
     assert(input != nullptr);
     DimType n, c, h, w;

ark/ops/ops_layernorm.cc

@@ -4,15 +4,14 @@
 #include "logging.h"
 #include "model.h"
 #include "tensor.h"
-
-using namespace std;
+#include <cassert>
 
 namespace ark {
 
 extern const OpConfigMap LayernormConfigMap;
 
 LayernormOp::LayernormOp(OpPrecType prec_type, Tensor *input, Tensor *output,
-                         const string &name)
+                         const std::string &name)
     : Op{OP_LAYERNORM, prec_type,           {input}, {output}, {},
          name,         &LayernormConfigMap, -1}
 {
@@ -45,7 +44,7 @@ std::string LayernormOp::function_name(const OpConfig &cfg) const
                              }});
 }
 
-Tensor *Model::layernorm(Tensor *input, Tensor *output, const string &name)
+Tensor *Model::layernorm(Tensor *input, Tensor *output, const std::string &name)
 {
     assert(input != nullptr);
     LOG(DEBUG, "layernorm ", input->shape, " ", input->ldims, " ");

ark/ops/ops_matmul_test.cc

@@ -6,6 +6,7 @@
 #include "include/ark_utils.h"
 #include "logging.h"
 #include "unittest/unittest_utils.h"
+#include <cassert>
 
 using namespace std;
 

ark/ops/ops_max_pool.cc

@@ -4,20 +4,21 @@
 #include "logging.h"
 #include "model.h"
 #include "tensor.h"
-
-using namespace std;
+#include <cassert>
 
 namespace ark {
 
 MaxPoolOp::MaxPoolOp(OpPrecType prec_type, Tensor *input, Tensor *output,
-                     DimType kernel_size, DimType stride, const string &name)
+                     DimType kernel_size, DimType stride,
+                     const std::string &name)
     : Op{OP_MAX_POOL, prec_type, {input}, {output}, {{kernel_size, stride}},
          name,        nullptr,   -1}
 {
 }
 
+// TODO: implement
 Tensor *Model::max_pool(Tensor *input, DimType kernel_size, DimType stride,
-                        Tensor *output, const string &name)
+                        Tensor *output, const std::string &name)
 {
     assert(input != nullptr);
     OpPrecType pt;

ark/ops/ops_mul.cc

@@ -4,15 +4,14 @@
 #include "logging.h"
 #include "model.h"
 #include "tensor.h"
-
-using namespace std;
+#include <cassert>
 
 namespace ark {
 
 extern const OpConfigMap ArithmeticConfigMap;
 
 MulOp::MulOp(OpPrecType prec_type, Tensor *input, Tensor *other, Tensor *output,
-             const string &name)
+             const std::string &name)
     : Op{OP_MUL, prec_type, {input, other},       {output},
          {},     name,      &ArithmeticConfigMap, -1,
          true}
@@ -49,7 +48,7 @@ std::string MulOp::function_name(const OpConfig &cfg) const
 }
 
 Tensor *Model::mul(Tensor *input, Tensor *other, Tensor *output,
-                   const string &name)
+                   const std::string &name)
 {
     LOG(DEBUG, "mul ", input->shape, " ", other->shape);
     assert(input != nullptr);