VirtualArray.h
Go to the documentation of this file.
1 // BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
2 
3 #ifndef AWKWARD_VIRTUALARRAY_H_
4 #define AWKWARD_VIRTUALARRAY_H_
5 
6 #include <string>
7 #include <memory>
8 #include <vector>
9 
10 #include "awkward/common.h"
11 #include "awkward/Slice.h"
12 #include "awkward/Content.h"
13 #include "awkward/virtual/ArrayGenerator.h"
14 #include "awkward/virtual/ArrayCache.h"
15 
16 namespace awkward {
20  class LIBAWKWARD_EXPORT_SYMBOL VirtualForm: public Form {
21  public:
23  VirtualForm(bool has_identities,
24  const util::Parameters& parameters,
25  const FormKey& form_key,
26  const FormPtr& form,
27  bool has_length);
28 
29  bool
30  has_form() const;
31 
32  const FormPtr
33  form() const;
34 
35  bool
36  has_length() const;
37 
38  const TypePtr
39  type(const util::TypeStrs& typestrs) const override;
40 
41  void
42  tojson_part(ToJson& builder, bool verbose) const override;
43 
44  const FormPtr
45  shallow_copy() const override;
46 
47  const FormPtr
48  with_form_key(const FormKey& form_key) const override;
49 
50  const std::string
51  purelist_parameter(const std::string& key) const override;
52 
53  bool
54  purelist_isregular() const override;
55 
56  int64_t
57  purelist_depth() const override;
58 
59  bool
60  dimension_optiontype() const override;
61 
62  const std::pair<int64_t, int64_t>
63  minmax_depth() const override;
64 
65  const std::pair<bool, int64_t>
66  branch_depth() const override;
67 
68  int64_t
69  numfields() const override;
70 
71  int64_t
72  fieldindex(const std::string& key) const override;
73 
74  const std::string
75  key(int64_t fieldindex) const override;
76 
77  bool
78  haskey(const std::string& key) const override;
79 
80  const std::vector<std::string>
81  keys() const override;
82 
83  bool
84  istuple() const override;
85 
86  bool
87  equal(const FormPtr& other,
88  bool check_identities,
89  bool check_parameters,
90  bool check_form_key,
91  bool compatibility_check) const override;
92 
93  const FormPtr
94  getitem_field(const std::string& key) const override;
95 
96  const FormPtr
97  getitem_fields(const std::vector<std::string>& keys) const override;
98 
99  private:
100  const FormPtr form_;
101  bool has_length_;
102  };
103 
109  class LIBAWKWARD_EXPORT_SYMBOL VirtualArray: public Content {
110  public:
123  VirtualArray(const IdentitiesPtr& identities,
124  const util::Parameters& parameters,
125  const ArrayGeneratorPtr& generator,
126  const ArrayCachePtr& cache,
127  const std::string& cache_key,
128  const kernel::lib ptr_lib = kernel::lib::cpu);
129 
132  VirtualArray(const IdentitiesPtr& identities,
133  const util::Parameters& parameters,
134  const ArrayGeneratorPtr& generator,
135  const ArrayCachePtr& cache,
136  const kernel::lib ptr_lib = kernel::lib::cpu);
137 
140  const ArrayGeneratorPtr
141  generator() const;
142 
145  const ArrayCachePtr
146  cache() const;
147 
148  const kernel::lib
149  ptr_lib() const;
150 
155  const ContentPtr
156  peek_array() const;
157 
161  const ContentPtr
162  array() const;
163 
165  const std::string
166  cache_key() const;
167 
169  const std::string
170  classname() const override;
171 
172  void
173  setidentities() override;
174 
175  void
176  setidentities(const IdentitiesPtr& identities) override;
177 
178  const TypePtr
179  type(const util::TypeStrs& typestrs) const override;
180 
181  const FormPtr
182  form(bool materialize) const override;
183 
184  kernel::lib
185  kernels() const override;
186 
187  void
188  caches(std::vector<ArrayCachePtr>& out) const override;
189 
190  const std::string
191  tostring_part(const std::string& indent,
192  const std::string& pre,
193  const std::string& post) const override;
194 
195  void
196  tojson_part(ToJson& builder, bool include_beginendlist) const override;
197 
202  void
203  nbytes_part(std::map<size_t, int64_t>& largest) const override;
204 
205  int64_t
206  length() const override;
207 
208  const ContentPtr
209  shallow_copy() const override;
210 
211  const ContentPtr
212  deep_copy(bool copyarrays,
213  bool copyindexes,
214  bool copyidentities) const override;
215 
216  void
217  check_for_iteration() const override;
218 
219  const ContentPtr
220  getitem_nothing() const override;
221 
222  const ContentPtr
223  getitem_at(int64_t at) const override;
224 
225  const ContentPtr
226  getitem_at_nowrap(int64_t at) const override;
227 
228  const ContentPtr
229  getitem_range(int64_t start, int64_t stop) const override;
230 
231  const ContentPtr
232  getitem_range_nowrap(int64_t start, int64_t stop) const override;
233 
234  const ContentPtr
235  getitem_field(const std::string& key) const override;
236 
237  const ContentPtr
238  getitem_field(const std::string& key,
239  const Slice& only_fields) const override;
240 
241  const ContentPtr
242  getitem_fields(const std::vector<std::string>& keys) const override;
243 
244  const ContentPtr
245  getitem_fields(const std::vector<std::string>& keys,
246  const Slice& only_fields) const override;
247 
248  const ContentPtr
249  getitem_next(const SliceItemPtr& head,
250  const Slice& tail,
251  const Index64& advanced) const override;
252 
253  const ContentPtr
254  getitem_next_jagged(const Index64& slicestarts,
255  const Index64& slicestops,
256  const SliceItemPtr& slicecontent,
257  const Slice& tail) const override;
258 
259  const ContentPtr
260  carry(const Index64& carry, bool allow_lazy) const override;
261 
262  int64_t
263  purelist_depth() const override;
264 
265  const std::pair<int64_t, int64_t>
266  minmax_depth() const override;
267 
268  const std::pair<bool, int64_t>
269  branch_depth() const override;
270 
271  int64_t
272  numfields() const override;
273 
274  int64_t
275  fieldindex(const std::string& key) const override;
276 
277  const std::string
278  key(int64_t fieldindex) const override;
279 
280  bool
281  haskey(const std::string& key) const override;
282 
283  const std::vector<std::string>
284  keys() const override;
285 
286  bool
287  istuple() const override;
288 
289  // operations
290  const std::string
291  validityerror(const std::string& path) const override;
292 
293  const ContentPtr
294  shallow_simplify() const override;
295 
296  const ContentPtr
297  num(int64_t axis, int64_t depth) const override;
298 
299  const std::pair<Index64, ContentPtr>
300  offsets_and_flattened(int64_t axis, int64_t depth) const override;
301 
302  bool
303  mergeable(const ContentPtr& other, bool mergebool) const override;
304 
305  bool
306  referentially_equal(const ContentPtr& other) const override;
307 
308  const ContentPtr
309  mergemany(const ContentPtrVec& others) const override;
310 
311  const SliceItemPtr
312  asslice() const override;
313 
314  const ContentPtr
315  fillna(const ContentPtr& value) const override;
316 
317  const ContentPtr
318  rpad(int64_t target, int64_t axis, int64_t depth) const override;
319 
320  const ContentPtr
321  rpad_and_clip(int64_t target,
322  int64_t axis,
323  int64_t depth) const override;
324 
325  const ContentPtr
326  reduce_next(const Reducer& reducer,
327  int64_t negaxis,
328  const Index64& starts,
329  const Index64& shifts,
330  const Index64& parents,
331  int64_t outlength,
332  bool mask,
333  bool keepdims) const override;
334 
335  const ContentPtr
336  sort_next(int64_t negaxis,
337  const Index64& starts,
338  const Index64& parents,
339  int64_t outlength,
340  bool ascending,
341  bool stable) const override;
342 
343  const ContentPtr
344  argsort_next(int64_t negaxis,
345  const Index64& starts,
346  const Index64& shifts,
347  const Index64& parents,
348  int64_t outlength,
349  bool ascending,
350  bool stable) const override;
351 
352 
353  const ContentPtr
354  localindex(int64_t axis, int64_t depth) const override;
355 
356  const ContentPtr
357  combinations(int64_t n,
358  bool replacement,
359  const util::RecordLookupPtr& recordlookup,
360  const util::Parameters& parameters,
361  int64_t axis,
362  int64_t depth) const override;
363 
364  const ContentPtr
365  getitem(const Slice& where) const override;
366 
367  const ContentPtr
368  getitem_next(const SliceAt& at,
369  const Slice& tail,
370  const Index64& advanced) const override;
371 
372  const ContentPtr
373  getitem_next(const SliceRange& range,
374  const Slice& tail,
375  const Index64& advanced) const override;
376 
377  const ContentPtr
378  getitem_next(const SliceArray64& array,
379  const Slice& tail,
380  const Index64& advanced) const override;
381 
382  const ContentPtr
383  getitem_next(const SliceField& field,
384  const Slice& tail,
385  const Index64& advanced) const override;
386 
387  const ContentPtr
388  getitem_next(const SliceFields& fields,
389  const Slice& tail,
390  const Index64& advanced) const override;
391 
392  const ContentPtr
393  getitem_next(const SliceJagged64& jagged,
394  const Slice& tail,
395  const Index64& advanced) const override;
396 
397  const ContentPtr
398  getitem_next_jagged(const Index64& slicestarts,
399  const Index64& slicestops,
400  const SliceArray64& slicecontent,
401  const Slice& tail) const override;
402 
403  const ContentPtr
404  getitem_next_jagged(const Index64& slicestarts,
405  const Index64& slicestops,
406  const SliceMissing64& slicecontent,
407  const Slice& tail) const override;
408 
409  const ContentPtr
410  getitem_next_jagged(const Index64& slicestarts,
411  const Index64& slicestops,
412  const SliceJagged64& slicecontent,
413  const Slice& tail) const override;
414 
415  const ContentPtr
416  copy_to(kernel::lib ptr_lib) const override;
417 
418  const ContentPtr
419  numbers_to_type(const std::string& name) const override;
420 
422  bool
423  is_unique() const override;
424 
426  const ContentPtr
427  unique() const override;
428 
430  bool
431  is_subrange_equal(const Index64& start, const Index64& stop) const override;
432 
433  private:
435  const util::Parameters
436  forward_parameters() const;
437 
440  void
441  set_cache_depths_from(const VirtualArray* original);
442 
445  void
446  set_cache_depths_from(const FormPtr& form);
447 
448  void
449  add_to_cache_depths(int64_t delta);
450 
452  const ArrayGeneratorPtr generator_;
454  const ArrayCachePtr cache_;
456  const std::string cache_key_;
458  const kernel::lib ptr_lib_;
459 
462  std::vector<int64_t> cache_depths_;
463  };
464 
465 }
466 
467 #endif // AWKWARD_VIRTUALARRAY_H_
awkward::VirtualArray
Represents an array that can be generated on demand.
Definition: VirtualArray.h:109
awkward::SliceRange
Represents a Python slice object (usual syntax: array[start:stop:step]).
Definition: Slice.h:93
LIBAWKWARD_EXPORT_SYMBOL
#define LIBAWKWARD_EXPORT_SYMBOL
Definition: common.h:45
awkward::util::key
const std::string key(const RecordLookupPtr &recordlookup, int64_t fieldindex, int64_t numfields)
Returns the key associated with a field index, given a RecordLookup and a number of fields...
awkward::Content
Abstract superclass of all array node types (flat hierarchy). Any Content can be nested within any ot...
Definition: Content.h:276
awkward::ToJson
Definition: json.h:19
awkward
Definition: BitMaskedArray.h:15
awkward::IndexOf
A contiguous, one-dimensional array of integers used to represent data structures, rather than numerical data in the arrays themselves.
Definition: Index.h:16
awkward::util::name
std::string name(Mapping a, V value)
Definition: datetime_util.h:39
awkward::SliceFields
Definition: Slice.h:384
awkward::SliceMissingOf
Represents a SliceArrayOf, SliceMissingOf, or SliceJaggedOf with missing values: None (no equivalent ...
Definition: Slice.h:431
awkward::SliceField
Represents a single string in a slice tuple, indicating that a RecordArray should be replaced by one ...
Definition: Slice.h:340
Content.h
awkward::Reducer
Abstract class for all reducer algorithms.
Definition: Reducer.h:20
awkward::kernel::combinations
ERROR combinations(kernel::lib ptr_lib, T *toindex, int64_t n, bool replacement, int64_t singlelen)
awkward::util::keys
const std::vector< std::string > keys(const RecordLookupPtr &recordlookup, int64_t numfields)
Returns a given RecordLookup as keys or generate anonymous ones form a number of fields.
awkward::ArrayCachePtr
std::shared_ptr< ArrayCache > ArrayCachePtr
Definition: Content.h:21
awkward::FormKey
std::shared_ptr< std::string > FormKey
Definition: Content.h:19
awkward::kernel::unique
ERROR unique(kernel::lib ptr_lib, T *toptr, int64_t length, int64_t *tolength)
awkward::SliceArrayOf
Represents an array of integers in a slice (possibly converted from an array of booleans).
Definition: Slice.h:225
awkward::util::Parameters
std::map< std::string, std::string > Parameters
Definition: util.h:165
awkward::kernel::copy_to
ERROR copy_to(kernel::lib to_lib, kernel::lib from_lib, void *to_ptr, void *from_ptr, int64_t bytelength)
Internal Function an array buffer from library FROM to library TO, usually between main memory and a ...
awkward::FormPtr
std::shared_ptr< Form > FormPtr
Definition: Content.h:18
awkward::util::fieldindex
int64_t fieldindex(const RecordLookupPtr &recordlookup, const std::string &key, int64_t numfields)
Returns the field index associated with a key, given a RecordLookup and a number of fields...
awkward::SliceJaggedOf
Represents an array of nested lists, where the content may be SliceArrayOf, SliceMissingOf, or SliceJaggedOf (no equivalent in NumPy).
Definition: Slice.h:511
awkward::ContentPtrVec
std::vector< std::shared_ptr< Content > > ContentPtrVec
Definition: Content.h:16
awkward::util::value
Mapping::value_type::value_type value(Mapping a, const std::string &name)
Definition: datetime_util.h:49
awkward::Slice
A sequence of SliceItem objects representing a tuple passed to Python&#39;s __getitem__.
Definition: Slice.h:585
awkward::util::haskey
bool haskey(const RecordLookupPtr &recordlookup, const std::string &key, int64_t numfields)
Returns true if a RecordLookup has a given key; false otherwise.
awkward::TypePtr
std::shared_ptr< Type > TypePtr
Definition: Content.h:23
awkward::util::RecordLookupPtr
std::shared_ptr< RecordLookup > RecordLookupPtr
Definition: util.h:130
awkward::SliceAt
Represents an integer in a tuple of slices passed to __getitem__ in Python.
Definition: Slice.h:58
awkward::util::TypeStrs
std::map< std::string, std::string > TypeStrs
Definition: util.h:215
awkward::ArrayGeneratorPtr
std::shared_ptr< ArrayGenerator > ArrayGeneratorPtr
Definition: ArrayGenerator.h:14
awkward::Form
Abstract superclass of all array node forms, which expresses the nesting structure without any large ...
Definition: Content.h:39
awkward::VirtualForm
Form describing VirtualArray.
Definition: VirtualArray.h:20
awkward::IdentitiesPtr
std::shared_ptr< Identities > IdentitiesPtr
Definition: Identities.h:16
awkward::ContentPtr
std::shared_ptr< Content > ContentPtr
Definition: ArrayBuilder.h:16
awkward::SliceItemPtr
std::shared_ptr< SliceItem > SliceItemPtr
Definition: Slice.h:15
awkward::kernel::lib
lib
Definition: kernel-dispatch.h:20