Content.h
Go to the documentation of this file.
1 // BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
2 
3 #ifndef AWKWARD_CONTENT_H_
4 #define AWKWARD_CONTENT_H_
5 
6 #include <map>
7 
8 #include "awkward/common.h"
9 #include "awkward/Identities.h"
10 #include "awkward/Slice.h"
11 #include "awkward/Index.h"
12 
13 namespace awkward {
14  class Content;
15  using ContentPtr = std::shared_ptr<Content>;
16  using ContentPtrVec = std::vector<std::shared_ptr<Content>>;
17  class Form;
18  using FormPtr = std::shared_ptr<Form>;
19  using FormKey = std::shared_ptr<std::string>;
20  class ArrayCache;
21  using ArrayCachePtr = std::shared_ptr<ArrayCache>;
22  class Type;
23  using TypePtr = std::shared_ptr<Type>;
24  class Reducer;
25  class ToJson;
26  class ToJsonPrettyString;
27  class ToJsonString;
28  class ToJsonPrettyFile;
29  class ToJsonFile;
30 
39  class LIBAWKWARD_EXPORT_SYMBOL Form {
40  public:
41  static FormPtr
42  fromnumpy(char kind,
43  int64_t itemsize,
44  const std::vector<int64_t>& inner_shape);
45  static FormPtr
46  fromjson(const std::string& data);
47 
50  Form(bool has_identities,
51  const util::Parameters& parameters,
52  const FormKey& form_key);
53 
55  virtual ~Form() { }
56 
61  virtual const TypePtr
62  type(const util::TypeStrs& typestrs) const = 0;
63 
66  virtual void
67  tojson_part(ToJson& builder, bool verbose) const = 0;
68 
71  virtual const FormPtr
72  shallow_copy() const = 0;
73 
75  virtual const FormPtr
76  with_form_key(const FormKey& form_key) const = 0;
77 
91  virtual bool
92  equal(const FormPtr& other,
93  bool check_identities,
94  bool check_parameters,
95  bool check_form_key,
96  bool compatibility_check) const = 0;
97 
99  bool
100  form_key_equals(const FormKey& other_form_key) const;
101 
105  virtual const std::string
106  purelist_parameter(const std::string& key) const = 0;
107 
118  bool
119  parameter_equals(const std::string& key, const std::string& value) const;
120 
123  virtual bool
124  purelist_isregular() const = 0;
125 
131  virtual int64_t
132  purelist_depth() const = 0;
133 
136  virtual bool
137  dimension_optiontype() const = 0;
138 
141  const FormKey
142  form_key() const;
143 
148  virtual const std::pair<int64_t, int64_t>
149  minmax_depth() const = 0;
150 
157  virtual const std::pair<bool, int64_t>
158  branch_depth() const = 0;
159 
162  virtual int64_t
163  numfields() const = 0;
164 
167  virtual int64_t
168  fieldindex(const std::string& key) const = 0;
169 
174  virtual const std::string
175  key(int64_t fieldindex) const = 0;
176 
179  virtual bool
180  haskey(const std::string& key) const = 0;
181 
184  virtual const std::vector<std::string>
185  keys() const = 0;
186 
189  virtual const std::string
190  tostring() const;
191 
196  virtual const std::string
197  tojson(bool pretty, bool verbose) const;
198 
201  bool
202  has_identities() const;
203 
210  const util::Parameters
211  parameters() const;
212 
220  const std::string
221  parameter(const std::string& key) const;
222 
226  void
227  identities_tojson(ToJson& builder, bool verbose) const;
228 
232  void
233  parameters_tojson(ToJson& builder, bool verbose) const;
234 
238  void
239  form_key_tojson(ToJson& builder, bool verbose) const;
240 
244  virtual const FormPtr
245  getitem_range() const;
246 
250  virtual const FormPtr
251  getitem_field(const std::string& key) const = 0;
252 
256  virtual const FormPtr
257  getitem_fields(const std::vector<std::string>& keys) const = 0;
258 
259  protected:
261  bool has_identities_;
263  util::Parameters parameters_;
265  FormKey form_key_;
266  };
267 
272  class LIBAWKWARD_EXPORT_SYMBOL Content {
273  public:
276  Content(const IdentitiesPtr& identities,
277  const util::Parameters& parameters);
278 
280  virtual ~Content() { }
281 
302  virtual bool
303  isscalar() const;
304 
307  virtual const std::string
308  classname() const = 0;
309 
312  virtual const IdentitiesPtr
313  identities() const;
314 
323  virtual void
324  setidentities() = 0;
325 
334  virtual void
335  setidentities(const IdentitiesPtr& identities) = 0;
336 
341  virtual const TypePtr
342  type(const util::TypeStrs& typestrs) const = 0;
343 
352  virtual const FormPtr
353  form(bool materialize) const = 0;
354 
359  virtual kernel::lib
360  kernels() const = 0;
361 
364  virtual void
365  caches(std::vector<ArrayCachePtr>& out) const = 0;
366 
373  virtual const std::string
374  tostring_part(const std::string& indent,
375  const std::string& pre,
376  const std::string& post) const = 0;
377 
380  virtual void
381  tojson_part(ToJson& builder, bool include_beginendlist) const = 0;
382 
392  virtual void
393  nbytes_part(std::map<size_t, int64_t>& largest) const = 0;
394 
396  virtual int64_t
397  length() const = 0;
398 
403  virtual const ContentPtr
404  shallow_copy() const = 0;
405 
419  virtual const ContentPtr
420  deep_copy(bool copyarrays,
421  bool copyindexes,
422  bool copyidentities) const = 0;
423 
426  virtual void
427  check_for_iteration() const = 0;
428 
431  virtual const ContentPtr
432  getitem_nothing() const = 0;
433 
439  virtual const ContentPtr
440  getitem_at(int64_t at) const = 0;
441 
446  virtual const ContentPtr
447  getitem_at_nowrap(int64_t at) const = 0;
448 
460  virtual const ContentPtr
461  getitem_range(int64_t start, int64_t stop) const = 0;
462 
470  virtual const ContentPtr
471  getitem_range_nowrap(int64_t start, int64_t stop) const = 0;
472 
475  virtual const ContentPtr
476  getitem_field(const std::string& key) const = 0;
477 
478  virtual const ContentPtr
479  getitem_field(const std::string& key,
480  const Slice& only_fields) const = 0;
481 
484  virtual const ContentPtr
485  getitem_fields(const std::vector<std::string>& keys) const = 0;
486 
487  virtual const ContentPtr
488  getitem_fields(const std::vector<std::string>& keys,
489  const Slice& only_fields) const = 0;
490 
493  virtual const ContentPtr
494  getitem(const Slice& where) const;
495 
508  virtual const ContentPtr
509  getitem_next(const SliceItemPtr& head,
510  const Slice& tail,
511  const Index64& advanced) const;
512 
522  virtual const ContentPtr
523  getitem_next_jagged(const Index64& slicestarts,
524  const Index64& slicestops,
525  const SliceItemPtr& slicecontent,
526  const Slice& tail) const;
527 
553  virtual const ContentPtr
554  carry(const Index64& carry, bool allow_lazy) const = 0;
555 
559  virtual const std::string
560  purelist_parameter(const std::string& key) const;
561 
564  bool
565  purelist_isregular() const;
566 
569  virtual bool
570  dimension_optiontype() const;
571 
580  virtual int64_t
581  purelist_depth() const = 0;
582 
587  virtual const std::pair<int64_t, int64_t>
588  minmax_depth() const = 0;
589 
596  virtual const std::pair<bool, int64_t>
597  branch_depth() const = 0;
598 
601  virtual int64_t
602  numfields() const = 0;
603 
606  virtual int64_t
607  fieldindex(const std::string& key) const = 0;
608 
613  virtual const std::string
614  key(int64_t fieldindex) const = 0;
615 
618  virtual bool
619  haskey(const std::string& key) const = 0;
620 
623  virtual const std::vector<std::string>
624  keys() const = 0;
625 
628  virtual const std::string
629  validityerror(const std::string& path) const = 0;
630 
638  virtual const ContentPtr
639  shallow_simplify() const = 0;
640 
652  virtual const ContentPtr
653  num(int64_t axis, int64_t depth) const = 0;
654 
668  virtual const std::pair<Index64, ContentPtr>
669  offsets_and_flattened(int64_t axis, int64_t depth) const = 0;
670 
680  virtual bool
681  mergeable(const ContentPtr& other, bool mergebool) const = 0;
682 
687  virtual bool
688  referentially_equal(const ContentPtr& other) const = 0;
689 
711  virtual const std::pair<ContentPtrVec, ContentPtrVec>
712  merging_strategy(const ContentPtrVec& others) const;
713 
719  virtual const ContentPtr
720  reverse_merge(const ContentPtr& other) const;
721 
724  const ContentPtr
725  merge(const ContentPtr& other) const;
726 
729  virtual const ContentPtr
730  mergemany(const ContentPtrVec& others) const = 0;
731 
734  virtual const SliceItemPtr
735  asslice() const = 0;
736 
742  virtual const ContentPtr
743  fillna(const ContentPtr& value) const = 0;
744 
758  virtual const ContentPtr
759  rpad(int64_t target, int64_t axis, int64_t depth) const = 0;
760 
774  virtual const ContentPtr
775  rpad_and_clip(int64_t target, int64_t axis, int64_t depth) const = 0;
776 
806  virtual const ContentPtr
807  reduce_next(const Reducer& reducer,
808  int64_t negaxis,
809  const Index64& starts,
810  const Index64& shifts,
811  const Index64& parents,
812  int64_t outlength,
813  bool mask,
814  bool keepdims) const = 0;
815 
835  virtual const ContentPtr
836  sort_next(int64_t negaxis,
837  const Index64& starts,
838  const Index64& parents,
839  int64_t outlength,
840  bool ascending,
841  bool stable) const = 0;
842 
865  virtual const ContentPtr
866  argsort_next(int64_t negaxis,
867  const Index64& starts,
868  const Index64& shifts,
869  const Index64& parents,
870  int64_t outlength,
871  bool ascending,
872  bool stable) const = 0;
873 
888  virtual const ContentPtr
889  localindex(int64_t axis, int64_t depth) const = 0;
890 
956  virtual const ContentPtr
957  combinations(int64_t n,
958  bool replacement,
959  const util::RecordLookupPtr& recordlookup,
960  const util::Parameters& parameters,
961  int64_t axis,
962  int64_t depth) const = 0;
963 
972  const std::string
973  tostring() const;
974 
992  const std::string
993  tojson(bool pretty, int64_t maxdecimals,
994  const char* nan_string = nullptr,
995  const char* infinity_string = nullptr,
996  const char* minus_infinity_string = nullptr,
997  const char* complex_real_string = nullptr,
998  const char* complex_imag_string = nullptr) const;
999 
1020  void
1021  tojson(FILE* destination,
1022  bool pretty,
1023  int64_t maxdecimals,
1024  int64_t buffersize,
1025  const char* nan_string = nullptr,
1026  const char* infinity_string = nullptr,
1027  const char* minus_infinity_string = nullptr,
1028  const char* complex_real_string = nullptr,
1029  const char* complex_imag_string = nullptr) const;
1030 
1034  int64_t
1035  nbytes() const;
1036 
1053  const ContentPtr
1054  reduce(const Reducer& reducer,
1055  int64_t axis,
1056  bool mask,
1057  bool keepdims) const;
1058 
1071  const ContentPtr
1072  sort(int64_t axis, bool ascending, bool stable) const;
1073 
1087  const ContentPtr
1088  argsort(int64_t axis, bool ascending, bool stable) const;
1089 
1096  const util::Parameters
1097  parameters() const;
1098 
1109  void
1110  setparameters(const util::Parameters& parameters);
1111 
1119  const std::string
1120  parameter(const std::string& key) const;
1121 
1132  void
1133  setparameter(const std::string& key, const std::string& value);
1134 
1145  bool
1146  parameter_equals(const std::string& key, const std::string& value) const;
1147 
1161  bool
1162  parameters_equal(const util::Parameters& other, bool check_all) const;
1163 
1166  bool
1167  parameter_isstring(const std::string& key) const;
1168 
1171  bool
1172  parameter_isname(const std::string& key) const;
1173 
1176  const std::string
1177  parameter_asstring(const std::string& key) const;
1178 
1183  const ContentPtr
1184  merge_as_union(const ContentPtr& other) const;
1185 
1195  const ContentPtr
1196  rpad_axis0(int64_t target, bool clip) const;
1197 
1202  const ContentPtr
1203  localindex_axis0() const;
1204 
1226  const ContentPtr
1227  combinations_axis0(int64_t n,
1228  bool replacement,
1229  const util::RecordLookupPtr& recordlookup,
1230  const util::Parameters& parameters) const;
1231 
1236  virtual const ContentPtr
1237  getitem_next(const SliceAt& at,
1238  const Slice& tail,
1239  const Index64& advanced) const = 0;
1240 
1245  virtual const ContentPtr
1246  getitem_next(const SliceRange& range,
1247  const Slice& tail,
1248  const Index64& advanced) const = 0;
1249 
1254  virtual const ContentPtr
1255  getitem_next(const SliceEllipsis& ellipsis,
1256  const Slice& tail,
1257  const Index64& advanced) const;
1258 
1263  virtual const ContentPtr
1264  getitem_next(const SliceNewAxis& newaxis,
1265  const Slice& tail,
1266  const Index64& advanced) const;
1267 
1272  virtual const ContentPtr
1273  getitem_next(const SliceArray64& array,
1274  const Slice& tail,
1275  const Index64& advanced) const = 0;
1276 
1281  virtual const ContentPtr
1282  getitem_next(const SliceField& field,
1283  const Slice& tail,
1284  const Index64& advanced) const;
1285 
1290  virtual const ContentPtr
1291  getitem_next(const SliceFields& fields,
1292  const Slice& tail,
1293  const Index64& advanced) const;
1294 
1299  virtual const ContentPtr
1300  getitem_next(const SliceMissing64& missing,
1301  const Slice& tail,
1302  const Index64& advanced) const;
1303 
1308  virtual const ContentPtr
1309  getitem_next(const SliceJagged64& jagged,
1310  const Slice& tail,
1311  const Index64& advanced) const = 0;
1312 
1317  virtual const ContentPtr
1318  getitem_next_jagged(const Index64& slicestarts,
1319  const Index64& slicestops,
1320  const SliceArray64& slicecontent,
1321  const Slice& tail) const = 0;
1322 
1327  virtual const ContentPtr
1328  getitem_next_jagged(const Index64& slicestarts,
1329  const Index64& slicestops,
1330  const SliceMissing64& slicecontent,
1331  const Slice& tail) const = 0;
1332 
1337  virtual const ContentPtr
1338  getitem_next_jagged(const Index64& slicestarts,
1339  const Index64& slicestops,
1340  const SliceJagged64& slicecontent,
1341  const Slice& tail) const = 0;
1342 
1357  const int64_t
1358  axis_wrap_if_negative(int64_t axis) const;
1359 
1363  virtual const ContentPtr
1364  copy_to(kernel::lib ptr_lib) const = 0;
1365 
1367  virtual const ContentPtr
1368  numbers_to_type(const std::string& name) const = 0;
1369 
1371  virtual bool
1372  is_unique() const = 0;
1373 
1376  virtual const ContentPtr
1377  unique() const = 0;
1378 
1380  virtual bool
1381  is_subrange_equal(const Index64& start, const Index64& stop) const = 0;
1382 
1383  protected:
1386  const std::string
1387  validityerror_parameters(const std::string& path) const;
1388 
1394  const ContentPtr
1395  getitem_next_array_wrap(const ContentPtr& outcontent,
1396  const std::vector<int64_t>& shape) const;
1397 
1405  const std::string
1406  parameters_tostring(const std::string& indent,
1407  const std::string& pre,
1408  const std::string& post) const;
1409 
1411  kernel::lib
1412  kernels_compare(kernel::lib from_index, const ContentPtr& content) const;
1413 
1414  protected:
1416  IdentitiesPtr identities_;
1418  util::Parameters parameters_;
1419  };
1420 }
1421 
1422 #endif // AWKWARD_CONTENT_H_
awkward::ToJsonPrettyFile
Produces a pretty JSON-formatted file.
Definition: json.h:265
awkward::ArrayCache
Abstract superclass of cache for VirtualArray, defining the interface.
Definition: ArrayCache.h:17
awkward::util::parameter_isstring
bool parameter_isstring(const Parameters &parameters, const std::string &key)
Returns true if the parameter associated with key is a string; false otherwise.
awkward::util::parameter_asstring
const std::string parameter_asstring(const Parameters &parameters, const std::string &key)
Returns the parameter associated with key as a string if parameter_isstring; raises an error otherwis...
awkward::SliceRange
Represents a Python slice object (usual syntax: array[start:stop:step]).
Definition: Slice.h:93
LIBAWKWARD_EXPORT_SYMBOL
#define LIBAWKWARD_EXPORT_SYMBOL
Definition: common.h:45
awkward::util::key
const std::string key(const RecordLookupPtr &recordlookup, int64_t fieldindex, int64_t numfields)
Returns the key associated with a field index, given a RecordLookup and a number of fields...
awkward::Content
Abstract superclass of all array node types (flat hierarchy). Any Content can be nested within any ot...
Definition: Content.h:272
awkward::ToJson
Definition: json.h:18
awkward
Definition: BitMaskedArray.h:15
awkward::IndexOf
A contiguous, one-dimensional array of integers used to represent data structures, rather than numerical data in the arrays themselves.
Definition: Index.h:16
awkward::SliceNewAxis
Represents NumPy&#39;s newaxis marker (a.k.a. None), which prompts __getitem__ to insert a length-1 regul...
Definition: Slice.h:197
awkward::util::name
std::string name(Mapping a, V value)
Definition: datetime_util.h:39
awkward::Type
Abstract superclass of all high level types (flat hierarchy).
Definition: Type.h:23
awkward::Form::form_key_
FormKey form_key_
See form_key.
Definition: Content.h:265
awkward::Form::parameters_
util::Parameters parameters_
See parameters.
Definition: Content.h:263
awkward::SliceFields
Definition: Slice.h:384
awkward::SliceMissingOf
Represents a SliceArrayOf, SliceMissingOf, or SliceJaggedOf with missing values: None (no equivalent ...
Definition: Slice.h:431
awkward::SliceField
Represents a single string in a slice tuple, indicating that a RecordArray should be replaced by one ...
Definition: Slice.h:340
awkward::Reducer
Abstract class for all reducer algorithms.
Definition: Reducer.h:20
awkward::kernel::combinations
ERROR combinations(kernel::lib ptr_lib, T *toindex, int64_t n, bool replacement, int64_t singlelen)
awkward::util::keys
const std::vector< std::string > keys(const RecordLookupPtr &recordlookup, int64_t numfields)
Returns a given RecordLookup as keys or generate anonymous ones form a number of fields.
awkward::ArrayCachePtr
std::shared_ptr< ArrayCache > ArrayCachePtr
Definition: Content.h:21
awkward::ToJsonString
Produces a JSON-formatted string.
Definition: json.h:72
awkward::Content::parameters_
util::Parameters parameters_
See parameters.
Definition: Content.h:1418
awkward::FormKey
std::shared_ptr< std::string > FormKey
Definition: Content.h:19
awkward::kernel::unique
ERROR unique(kernel::lib ptr_lib, T *toptr, int64_t length, int64_t *tolength)
awkward::SliceArrayOf
Represents an array of integers in a slice (possibly converted from an array of booleans).
Definition: Slice.h:225
awkward::util::Parameters
std::map< std::string, std::string > Parameters
Definition: util.h:165
awkward::kernel::copy_to
ERROR copy_to(kernel::lib to_lib, kernel::lib from_lib, void *to_ptr, void *from_ptr, int64_t bytelength)
Internal Function an array buffer from library FROM to library TO, usually between main memory and a ...
awkward::FormPtr
std::shared_ptr< Form > FormPtr
Definition: Content.h:18
awkward::Content::identities_
IdentitiesPtr identities_
See identities.
Definition: Content.h:1416
awkward::util::fieldindex
int64_t fieldindex(const RecordLookupPtr &recordlookup, const std::string &key, int64_t numfields)
Returns the field index associated with a key, given a RecordLookup and a number of fields...
awkward::SliceJaggedOf
Represents an array of nested lists, where the content may be SliceArrayOf, SliceMissingOf, or SliceJaggedOf (no equivalent in NumPy).
Definition: Slice.h:511
awkward::ToJsonPrettyString
Produces a pretty JSON-formatted string.
Definition: json.h:136
awkward::ContentPtrVec
std::vector< std::shared_ptr< Content > > ContentPtrVec
Definition: Content.h:16
awkward::util::value
Mapping::value_type::value_type value(Mapping a, const std::string &name)
Definition: datetime_util.h:49
awkward::Slice
A sequence of SliceItem objects representing a tuple passed to Python&#39;s __getitem__.
Definition: Slice.h:585
awkward::SliceEllipsis
Represents a Python Ellipsis object (usual syntax: array[...]).
Definition: Slice.h:169
awkward::util::parameter_equals
bool parameter_equals(const Parameters &parameters, const std::string &key, const std::string &value)
Returns true if the value associated with a key in parameters is equal to the specified value...
awkward::util::haskey
bool haskey(const RecordLookupPtr &recordlookup, const std::string &key, int64_t numfields)
Returns true if a RecordLookup has a given key; false otherwise.
awkward::TypePtr
std::shared_ptr< Type > TypePtr
Definition: Content.h:23
awkward::util::parameters_equal
bool parameters_equal(const Parameters &self, const Parameters &other, bool check_all)
Returns true if all key-value pairs in self is equal to all key-value pairs in other.
awkward::Content::~Content
virtual ~Content()
Empty destructor; required for some C++ reason.
Definition: Content.h:280
awkward::util::parameter_isname
bool parameter_isname(const Parameters &parameters, const std::string &key)
Returns true if the parameter associated with key is a string that matches [A-Za-z_][A-Za-z_0-9]*; fa...
awkward::util::RecordLookupPtr
std::shared_ptr< RecordLookup > RecordLookupPtr
Definition: util.h:130
awkward::SliceAt
Represents an integer in a tuple of slices passed to __getitem__ in Python.
Definition: Slice.h:58
awkward::util::TypeStrs
std::map< std::string, std::string > TypeStrs
Definition: util.h:215
awkward::Form
Abstract superclass of all array node forms, which expresses the nesting structure without any large ...
Definition: Content.h:39
awkward::IdentitiesPtr
std::shared_ptr< Identities > IdentitiesPtr
Definition: Identities.h:16
awkward::ContentPtr
std::shared_ptr< Content > ContentPtr
Definition: ArrayBuilder.h:16
awkward::ToJsonFile
Produces a JSON-formatted file.
Definition: json.h:200
awkward::SliceItemPtr
std::shared_ptr< SliceItem > SliceItemPtr
Definition: Slice.h:15
awkward::kernel::lib
lib
Definition: kernel-dispatch.h:20
awkward::Form::~Form
virtual ~Form()
Empty destructor; required for some C++ reason.
Definition: Content.h:55
awkward::Form::has_identities_
bool has_identities_
See has_identities.
Definition: Content.h:261