Content.h
Go to the documentation of this file.
1 // BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
2 
3 #ifndef AWKWARD_CONTENT_H_
4 #define AWKWARD_CONTENT_H_
5 
6 #include <map>
7 
8 #include "awkward/common.h"
9 #include "awkward/Identities.h"
10 #include "awkward/Slice.h"
11 #include "awkward/Index.h"
12 
13 namespace awkward {
14  class Content;
15  using ContentPtr = std::shared_ptr<Content>;
16  using ContentPtrVec = std::vector<std::shared_ptr<Content>>;
17  class Form;
18  using FormPtr = std::shared_ptr<Form>;
19  using FormKey = std::shared_ptr<std::string>;
20  class ArrayCache;
21  using ArrayCachePtr = std::shared_ptr<ArrayCache>;
22  class Type;
23  using TypePtr = std::shared_ptr<Type>;
24  class Reducer;
25  class ToJson;
26  class ToJsonPrettyString;
27  class ToJsonString;
28  class ToJsonPrettyFile;
29  class ToJsonFile;
30 
39  class LIBAWKWARD_EXPORT_SYMBOL Form {
40  public:
41  static FormPtr
42  fromnumpy(char kind,
43  int64_t itemsize,
44  const std::vector<int64_t>& inner_shape);
45  static FormPtr
46  fromjson(const std::string& data);
47 
50  Form(bool has_identities,
51  const util::Parameters& parameters,
52  const FormKey& form_key);
53 
55  virtual ~Form() { }
56 
61  virtual const TypePtr
62  type(const util::TypeStrs& typestrs) const = 0;
63 
66  virtual void
67  tojson_part(ToJson& builder, bool verbose) const = 0;
68 
71  virtual const FormPtr
72  shallow_copy() const = 0;
73 
75  virtual const FormPtr
76  with_form_key(const FormKey& form_key) const = 0;
77 
91  virtual bool
92  equal(const FormPtr& other,
93  bool check_identities,
94  bool check_parameters,
95  bool check_form_key,
96  bool compatibility_check) const = 0;
97 
99  bool
100  form_key_equals(const FormKey& other_form_key) const;
101 
105  virtual const std::string
106  purelist_parameter(const std::string& key) const = 0;
107 
118  bool
119  parameter_equals(const std::string& key, const std::string& value) const;
120 
123  virtual bool
124  purelist_isregular() const = 0;
125 
131  virtual int64_t
132  purelist_depth() const = 0;
133 
136  virtual bool
137  dimension_optiontype() const = 0;
138 
141  const FormKey
142  form_key() const;
143 
148  virtual const std::pair<int64_t, int64_t>
149  minmax_depth() const = 0;
150 
157  virtual const std::pair<bool, int64_t>
158  branch_depth() const = 0;
159 
162  virtual int64_t
163  numfields() const = 0;
164 
167  virtual int64_t
168  fieldindex(const std::string& key) const = 0;
169 
174  virtual const std::string
175  key(int64_t fieldindex) const = 0;
176 
179  virtual bool
180  haskey(const std::string& key) const = 0;
181 
184  virtual const std::vector<std::string>
185  keys() const = 0;
186 
188  virtual bool
189  istuple() const = 0;
190 
193  virtual const std::string
194  tostring() const;
195 
200  virtual const std::string
201  tojson(bool pretty, bool verbose) const;
202 
205  bool
206  has_identities() const;
207 
214  const util::Parameters
215  parameters() const;
216 
224  const std::string
225  parameter(const std::string& key) const;
226 
230  void
231  identities_tojson(ToJson& builder, bool verbose) const;
232 
236  void
237  parameters_tojson(ToJson& builder, bool verbose) const;
238 
242  void
243  form_key_tojson(ToJson& builder, bool verbose) const;
244 
248  virtual const FormPtr
249  getitem_range() const;
250 
254  virtual const FormPtr
255  getitem_field(const std::string& key) const = 0;
256 
260  virtual const FormPtr
261  getitem_fields(const std::vector<std::string>& keys) const = 0;
262 
263  protected:
265  bool has_identities_;
267  util::Parameters parameters_;
269  FormKey form_key_;
270  };
271 
276  class LIBAWKWARD_EXPORT_SYMBOL Content {
277  public:
280  Content(const IdentitiesPtr& identities,
281  const util::Parameters& parameters);
282 
284  virtual ~Content() { }
285 
306  virtual bool
307  isscalar() const;
308 
311  virtual const std::string
312  classname() const = 0;
313 
316  virtual const IdentitiesPtr
317  identities() const;
318 
327  virtual void
328  setidentities() = 0;
329 
338  virtual void
339  setidentities(const IdentitiesPtr& identities) = 0;
340 
345  virtual const TypePtr
346  type(const util::TypeStrs& typestrs) const = 0;
347 
356  virtual const FormPtr
357  form(bool materialize) const = 0;
358 
363  virtual kernel::lib
364  kernels() const = 0;
365 
368  virtual void
369  caches(std::vector<ArrayCachePtr>& out) const = 0;
370 
377  virtual const std::string
378  tostring_part(const std::string& indent,
379  const std::string& pre,
380  const std::string& post) const = 0;
381 
384  virtual void
385  tojson_part(ToJson& builder, bool include_beginendlist) const = 0;
386 
396  virtual void
397  nbytes_part(std::map<size_t, int64_t>& largest) const = 0;
398 
400  virtual int64_t
401  length() const = 0;
402 
407  virtual const ContentPtr
408  shallow_copy() const = 0;
409 
423  virtual const ContentPtr
424  deep_copy(bool copyarrays,
425  bool copyindexes,
426  bool copyidentities) const = 0;
427 
430  virtual void
431  check_for_iteration() const = 0;
432 
435  virtual const ContentPtr
436  getitem_nothing() const = 0;
437 
443  virtual const ContentPtr
444  getitem_at(int64_t at) const = 0;
445 
450  virtual const ContentPtr
451  getitem_at_nowrap(int64_t at) const = 0;
452 
464  virtual const ContentPtr
465  getitem_range(int64_t start, int64_t stop) const = 0;
466 
474  virtual const ContentPtr
475  getitem_range_nowrap(int64_t start, int64_t stop) const = 0;
476 
479  virtual const ContentPtr
480  getitem_field(const std::string& key) const = 0;
481 
482  virtual const ContentPtr
483  getitem_field(const std::string& key,
484  const Slice& only_fields) const = 0;
485 
488  virtual const ContentPtr
489  getitem_fields(const std::vector<std::string>& keys) const = 0;
490 
491  virtual const ContentPtr
492  getitem_fields(const std::vector<std::string>& keys,
493  const Slice& only_fields) const = 0;
494 
497  virtual const ContentPtr
498  getitem(const Slice& where) const;
499 
512  virtual const ContentPtr
513  getitem_next(const SliceItemPtr& head,
514  const Slice& tail,
515  const Index64& advanced) const;
516 
526  virtual const ContentPtr
527  getitem_next_jagged(const Index64& slicestarts,
528  const Index64& slicestops,
529  const SliceItemPtr& slicecontent,
530  const Slice& tail) const;
531 
557  virtual const ContentPtr
558  carry(const Index64& carry, bool allow_lazy) const = 0;
559 
563  virtual const std::string
564  purelist_parameter(const std::string& key) const;
565 
568  bool
569  purelist_isregular() const;
570 
573  virtual bool
574  dimension_optiontype() const;
575 
584  virtual int64_t
585  purelist_depth() const = 0;
586 
591  virtual const std::pair<int64_t, int64_t>
592  minmax_depth() const = 0;
593 
600  virtual const std::pair<bool, int64_t>
601  branch_depth() const = 0;
602 
605  virtual int64_t
606  numfields() const = 0;
607 
610  virtual int64_t
611  fieldindex(const std::string& key) const = 0;
612 
617  virtual const std::string
618  key(int64_t fieldindex) const = 0;
619 
622  virtual bool
623  haskey(const std::string& key) const = 0;
624 
627  virtual const std::vector<std::string>
628  keys() const = 0;
629 
631  virtual bool
632  istuple() const = 0;
633 
636  virtual const std::string
637  validityerror(const std::string& path) const = 0;
638 
646  virtual const ContentPtr
647  shallow_simplify() const = 0;
648 
660  virtual const ContentPtr
661  num(int64_t axis, int64_t depth) const = 0;
662 
676  virtual const std::pair<Index64, ContentPtr>
677  offsets_and_flattened(int64_t axis, int64_t depth) const = 0;
678 
688  virtual bool
689  mergeable(const ContentPtr& other, bool mergebool) const = 0;
690 
695  virtual bool
696  referentially_equal(const ContentPtr& other) const = 0;
697 
719  virtual const std::pair<ContentPtrVec, ContentPtrVec>
720  merging_strategy(const ContentPtrVec& others) const;
721 
727  virtual const ContentPtr
728  reverse_merge(const ContentPtr& other) const;
729 
732  const ContentPtr
733  merge(const ContentPtr& other) const;
734 
737  virtual const ContentPtr
738  mergemany(const ContentPtrVec& others) const = 0;
739 
742  virtual const SliceItemPtr
743  asslice() const = 0;
744 
750  virtual const ContentPtr
751  fillna(const ContentPtr& value) const = 0;
752 
766  virtual const ContentPtr
767  rpad(int64_t target, int64_t axis, int64_t depth) const = 0;
768 
782  virtual const ContentPtr
783  rpad_and_clip(int64_t target, int64_t axis, int64_t depth) const = 0;
784 
814  virtual const ContentPtr
815  reduce_next(const Reducer& reducer,
816  int64_t negaxis,
817  const Index64& starts,
818  const Index64& shifts,
819  const Index64& parents,
820  int64_t outlength,
821  bool mask,
822  bool keepdims) const = 0;
823 
843  virtual const ContentPtr
844  sort_next(int64_t negaxis,
845  const Index64& starts,
846  const Index64& parents,
847  int64_t outlength,
848  bool ascending,
849  bool stable) const = 0;
850 
873  virtual const ContentPtr
874  argsort_next(int64_t negaxis,
875  const Index64& starts,
876  const Index64& shifts,
877  const Index64& parents,
878  int64_t outlength,
879  bool ascending,
880  bool stable) const = 0;
881 
896  virtual const ContentPtr
897  localindex(int64_t axis, int64_t depth) const = 0;
898 
964  virtual const ContentPtr
965  combinations(int64_t n,
966  bool replacement,
967  const util::RecordLookupPtr& recordlookup,
968  const util::Parameters& parameters,
969  int64_t axis,
970  int64_t depth) const = 0;
971 
980  const std::string
981  tostring() const;
982 
1000  const std::string
1001  tojson(bool pretty, int64_t maxdecimals,
1002  const char* nan_string = nullptr,
1003  const char* infinity_string = nullptr,
1004  const char* minus_infinity_string = nullptr,
1005  const char* complex_real_string = nullptr,
1006  const char* complex_imag_string = nullptr) const;
1007 
1028  void
1029  tojson(FILE* destination,
1030  bool pretty,
1031  int64_t maxdecimals,
1032  int64_t buffersize,
1033  const char* nan_string = nullptr,
1034  const char* infinity_string = nullptr,
1035  const char* minus_infinity_string = nullptr,
1036  const char* complex_real_string = nullptr,
1037  const char* complex_imag_string = nullptr) const;
1038 
1042  int64_t
1043  nbytes() const;
1044 
1061  const ContentPtr
1062  reduce(const Reducer& reducer,
1063  int64_t axis,
1064  bool mask,
1065  bool keepdims) const;
1066 
1079  const ContentPtr
1080  sort(int64_t axis, bool ascending, bool stable) const;
1081 
1095  const ContentPtr
1096  argsort(int64_t axis, bool ascending, bool stable) const;
1097 
1104  const util::Parameters
1105  parameters() const;
1106 
1117  void
1118  setparameters(const util::Parameters& parameters);
1119 
1127  const std::string
1128  parameter(const std::string& key) const;
1129 
1140  void
1141  setparameter(const std::string& key, const std::string& value);
1142 
1153  bool
1154  parameter_equals(const std::string& key, const std::string& value) const;
1155 
1169  bool
1170  parameters_equal(const util::Parameters& other, bool check_all) const;
1171 
1174  bool
1175  parameter_isstring(const std::string& key) const;
1176 
1179  bool
1180  parameter_isname(const std::string& key) const;
1181 
1184  const std::string
1185  parameter_asstring(const std::string& key) const;
1186 
1191  const ContentPtr
1192  merge_as_union(const ContentPtr& other) const;
1193 
1203  const ContentPtr
1204  rpad_axis0(int64_t target, bool clip) const;
1205 
1210  const ContentPtr
1211  localindex_axis0() const;
1212 
1234  const ContentPtr
1235  combinations_axis0(int64_t n,
1236  bool replacement,
1237  const util::RecordLookupPtr& recordlookup,
1238  const util::Parameters& parameters) const;
1239 
1244  virtual const ContentPtr
1245  getitem_next(const SliceAt& at,
1246  const Slice& tail,
1247  const Index64& advanced) const = 0;
1248 
1253  virtual const ContentPtr
1254  getitem_next(const SliceRange& range,
1255  const Slice& tail,
1256  const Index64& advanced) const = 0;
1257 
1262  virtual const ContentPtr
1263  getitem_next(const SliceEllipsis& ellipsis,
1264  const Slice& tail,
1265  const Index64& advanced) const;
1266 
1271  virtual const ContentPtr
1272  getitem_next(const SliceNewAxis& newaxis,
1273  const Slice& tail,
1274  const Index64& advanced) const;
1275 
1280  virtual const ContentPtr
1281  getitem_next(const SliceArray64& array,
1282  const Slice& tail,
1283  const Index64& advanced) const = 0;
1284 
1289  virtual const ContentPtr
1290  getitem_next(const SliceField& field,
1291  const Slice& tail,
1292  const Index64& advanced) const;
1293 
1298  virtual const ContentPtr
1299  getitem_next(const SliceFields& fields,
1300  const Slice& tail,
1301  const Index64& advanced) const;
1302 
1307  virtual const ContentPtr
1308  getitem_next(const SliceMissing64& missing,
1309  const Slice& tail,
1310  const Index64& advanced) const;
1311 
1316  virtual const ContentPtr
1317  getitem_next(const SliceJagged64& jagged,
1318  const Slice& tail,
1319  const Index64& advanced) const = 0;
1320 
1325  virtual const ContentPtr
1326  getitem_next_jagged(const Index64& slicestarts,
1327  const Index64& slicestops,
1328  const SliceArray64& slicecontent,
1329  const Slice& tail) const = 0;
1330 
1335  virtual const ContentPtr
1336  getitem_next_jagged(const Index64& slicestarts,
1337  const Index64& slicestops,
1338  const SliceMissing64& slicecontent,
1339  const Slice& tail) const = 0;
1340 
1345  virtual const ContentPtr
1346  getitem_next_jagged(const Index64& slicestarts,
1347  const Index64& slicestops,
1348  const SliceJagged64& slicecontent,
1349  const Slice& tail) const = 0;
1350 
1365  const int64_t
1366  axis_wrap_if_negative(int64_t axis) const;
1367 
1371  virtual const ContentPtr
1372  copy_to(kernel::lib ptr_lib) const = 0;
1373 
1375  virtual const ContentPtr
1376  numbers_to_type(const std::string& name) const = 0;
1377 
1379  virtual bool
1380  is_unique() const = 0;
1381 
1384  virtual const ContentPtr
1385  unique() const = 0;
1386 
1388  virtual bool
1389  is_subrange_equal(const Index64& start, const Index64& stop) const = 0;
1390 
1391  protected:
1394  const std::string
1395  validityerror_parameters(const std::string& path) const;
1396 
1402  const ContentPtr
1403  getitem_next_array_wrap(const ContentPtr& outcontent,
1404  const std::vector<int64_t>& shape) const;
1405 
1413  const std::string
1414  parameters_tostring(const std::string& indent,
1415  const std::string& pre,
1416  const std::string& post) const;
1417 
1419  kernel::lib
1420  kernels_compare(kernel::lib from_index, const ContentPtr& content) const;
1421 
1422  protected:
1424  IdentitiesPtr identities_;
1426  util::Parameters parameters_;
1427  };
1428 }
1429 
1430 #endif // AWKWARD_CONTENT_H_
awkward::ToJsonPrettyFile
Produces a pretty JSON-formatted file.
Definition: json.h:266
awkward::ArrayCache
Abstract superclass of cache for VirtualArray, defining the interface.
Definition: ArrayCache.h:17
awkward::util::parameter_isstring
bool parameter_isstring(const Parameters &parameters, const std::string &key)
Returns true if the parameter associated with key is a string; false otherwise.
awkward::util::parameter_asstring
const std::string parameter_asstring(const Parameters &parameters, const std::string &key)
Returns the parameter associated with key as a string if parameter_isstring; raises an error otherwis...
awkward::SliceRange
Represents a Python slice object (usual syntax: array[start:stop:step]).
Definition: Slice.h:93
LIBAWKWARD_EXPORT_SYMBOL
#define LIBAWKWARD_EXPORT_SYMBOL
Definition: common.h:45
awkward::util::key
const std::string key(const RecordLookupPtr &recordlookup, int64_t fieldindex, int64_t numfields)
Returns the key associated with a field index, given a RecordLookup and a number of fields...
awkward::Content
Abstract superclass of all array node types (flat hierarchy). Any Content can be nested within any ot...
Definition: Content.h:276
awkward::ToJson
Definition: json.h:19
awkward
Definition: BitMaskedArray.h:15
awkward::IndexOf
A contiguous, one-dimensional array of integers used to represent data structures, rather than numerical data in the arrays themselves.
Definition: Index.h:16
awkward::SliceNewAxis
Represents NumPy&#39;s newaxis marker (a.k.a. None), which prompts __getitem__ to insert a length-1 regul...
Definition: Slice.h:197
awkward::util::name
std::string name(Mapping a, V value)
Definition: datetime_util.h:39
awkward::Type
Abstract superclass of all high level types (flat hierarchy).
Definition: Type.h:23
awkward::Form::form_key_
FormKey form_key_
See form_key.
Definition: Content.h:269
awkward::Form::parameters_
util::Parameters parameters_
See parameters.
Definition: Content.h:267
awkward::SliceFields
Definition: Slice.h:384
awkward::SliceMissingOf
Represents a SliceArrayOf, SliceMissingOf, or SliceJaggedOf with missing values: None (no equivalent ...
Definition: Slice.h:431
awkward::SliceField
Represents a single string in a slice tuple, indicating that a RecordArray should be replaced by one ...
Definition: Slice.h:340
awkward::Reducer
Abstract class for all reducer algorithms.
Definition: Reducer.h:20
awkward::kernel::combinations
ERROR combinations(kernel::lib ptr_lib, T *toindex, int64_t n, bool replacement, int64_t singlelen)
awkward::util::keys
const std::vector< std::string > keys(const RecordLookupPtr &recordlookup, int64_t numfields)
Returns a given RecordLookup as keys or generate anonymous ones form a number of fields.
awkward::ArrayCachePtr
std::shared_ptr< ArrayCache > ArrayCachePtr
Definition: Content.h:21
awkward::ToJsonString
Produces a JSON-formatted string.
Definition: json.h:73
awkward::Content::parameters_
util::Parameters parameters_
See parameters.
Definition: Content.h:1426
awkward::FormKey
std::shared_ptr< std::string > FormKey
Definition: Content.h:19
awkward::kernel::unique
ERROR unique(kernel::lib ptr_lib, T *toptr, int64_t length, int64_t *tolength)
awkward::SliceArrayOf
Represents an array of integers in a slice (possibly converted from an array of booleans).
Definition: Slice.h:225
awkward::util::Parameters
std::map< std::string, std::string > Parameters
Definition: util.h:165
awkward::kernel::copy_to
ERROR copy_to(kernel::lib to_lib, kernel::lib from_lib, void *to_ptr, void *from_ptr, int64_t bytelength)
Internal Function an array buffer from library FROM to library TO, usually between main memory and a ...
awkward::FormPtr
std::shared_ptr< Form > FormPtr
Definition: Content.h:18
awkward::Content::identities_
IdentitiesPtr identities_
See identities.
Definition: Content.h:1424
awkward::util::fieldindex
int64_t fieldindex(const RecordLookupPtr &recordlookup, const std::string &key, int64_t numfields)
Returns the field index associated with a key, given a RecordLookup and a number of fields...
awkward::SliceJaggedOf
Represents an array of nested lists, where the content may be SliceArrayOf, SliceMissingOf, or SliceJaggedOf (no equivalent in NumPy).
Definition: Slice.h:511
awkward::ToJsonPrettyString
Produces a pretty JSON-formatted string.
Definition: json.h:137
awkward::ContentPtrVec
std::vector< std::shared_ptr< Content > > ContentPtrVec
Definition: Content.h:16
awkward::util::value
Mapping::value_type::value_type value(Mapping a, const std::string &name)
Definition: datetime_util.h:49
awkward::Slice
A sequence of SliceItem objects representing a tuple passed to Python&#39;s __getitem__.
Definition: Slice.h:585
awkward::SliceEllipsis
Represents a Python Ellipsis object (usual syntax: array[...]).
Definition: Slice.h:169
awkward::util::parameter_equals
bool parameter_equals(const Parameters &parameters, const std::string &key, const std::string &value)
Returns true if the value associated with a key in parameters is equal to the specified value...
awkward::util::haskey
bool haskey(const RecordLookupPtr &recordlookup, const std::string &key, int64_t numfields)
Returns true if a RecordLookup has a given key; false otherwise.
awkward::TypePtr
std::shared_ptr< Type > TypePtr
Definition: Content.h:23
awkward::util::parameters_equal
bool parameters_equal(const Parameters &self, const Parameters &other, bool check_all)
Returns true if all key-value pairs in self is equal to all key-value pairs in other.
awkward::Content::~Content
virtual ~Content()
Empty destructor; required for some C++ reason.
Definition: Content.h:284
awkward::util::parameter_isname
bool parameter_isname(const Parameters &parameters, const std::string &key)
Returns true if the parameter associated with key is a string that matches [A-Za-z_][A-Za-z_0-9]*; fa...
awkward::util::RecordLookupPtr
std::shared_ptr< RecordLookup > RecordLookupPtr
Definition: util.h:130
awkward::SliceAt
Represents an integer in a tuple of slices passed to __getitem__ in Python.
Definition: Slice.h:58
awkward::util::TypeStrs
std::map< std::string, std::string > TypeStrs
Definition: util.h:215
awkward::Form
Abstract superclass of all array node forms, which expresses the nesting structure without any large ...
Definition: Content.h:39
awkward::IdentitiesPtr
std::shared_ptr< Identities > IdentitiesPtr
Definition: Identities.h:16
awkward::ContentPtr
std::shared_ptr< Content > ContentPtr
Definition: ArrayBuilder.h:16
awkward::ToJsonFile
Produces a JSON-formatted file.
Definition: json.h:201
awkward::SliceItemPtr
std::shared_ptr< SliceItem > SliceItemPtr
Definition: Slice.h:15
awkward::kernel::lib
lib
Definition: kernel-dispatch.h:20
awkward::Form::~Form
virtual ~Form()
Empty destructor; required for some C++ reason.
Definition: Content.h:55
awkward::Form::has_identities_
bool has_identities_
See has_identities.
Definition: Content.h:265