CAPIO-CL 2.0.3
CAPIO-CL: Cross Application Programmable I/O - Coordination Language
Loading...
Searching...
No Matches
engine.h
1#ifndef CAPIO_CL_ENGINE_H
2#define CAPIO_CL_ENGINE_H
3#include <jsoncons/basic_json.hpp>
4#include <shared_mutex>
5#include <vector>
6
7#include "capiocl.hpp"
8#include "capiocl/api.h"
9#include "capiocl/monitor.h"
10#include "capiocl/serializer.h"
11
13namespace capiocl::engine {
14
17struct CapioCLEntry final {
18 // LCOV_EXCL_START
20 std::vector<std::string> producers;
22 std::vector<std::string> consumers;
24 std::vector<std::filesystem::path> file_dependencies;
36 bool store_in_memory = false;
38 bool permanent = false;
40 bool excluded = false;
42 bool is_file = true;
43 // LCOV_EXCL_STOP
44
50 static CapioCLEntry fromJson(const std::string &in);
51
53 [[nodiscard]] std::string toJson() const;
54
57
60
62 bool operator==(const CapioCLEntry &other);
63
65 bool operator!=(const CapioCLEntry &other);
66};
67
83class Engine final {
84 friend class serializer::Serializer;
85
87 mutable std::shared_mutex _shared_mutex;
88
90 bool store_all_in_memory = false;
91
94
96 void configureMonitorBackends();
97
99 monitor::Monitor monitor;
100
102 std::string node_name;
103
105 std::string workflow_name;
106
108 std::unique_ptr<api::CapioClApiServer> webapi_server;
109
111 mutable std::unordered_map<std::string, CapioCLEntry> _capio_cl_entries;
112
120 mutable std::unordered_map<std::string, std::string> _glob_patterns;
121
127 mutable std::unordered_map<std::string, std::vector<std::string>> _normalized_aliases;
128
134 static bool _isGlobPattern(const std::string &path);
135
141 void _addToGlobIndex(const std::string &path) const;
142
148 void _removeFromGlobIndex(const std::string &path) const;
149
159 bool _isAppListed(const std::string &path, const std::string &app_name,
160 std::vector<std::string> CapioCLEntry::*list) const;
161
172 static std::string truncateLastN(const std::string &str, const std::size_t n) {
173 if (str.length() > n) {
174 return "[..] " + str.substr(str.length() - n);
175 }
176 return str;
177 }
178
183 void _newFile(const std::filesystem::path &path) const;
184
197 void compute_directory_entry_count(const std::filesystem::path &path) const;
198
199 public:
201 explicit Engine(bool use_default_settings = true);
202
204 explicit Engine(const configuration::CapioClConfiguration &config);
205
207 void print() const;
208
216 bool contains(const std::filesystem::path &file) const;
217
219 size_t size() const;
220
233 void add(std::filesystem::path &path, std::vector<std::string> &producers,
234 std::vector<std::string> &consumers, const std::string &commit_rule,
235 const std::string &fire_rule, bool permanent, bool exclude,
236 std::vector<std::filesystem::path> &dependencies);
237
243 void add(const std::filesystem::path &path, const CapioCLEntry &entry) const;
244
251 void addProducer(const std::filesystem::path &path, std::string &producer);
252
259 void addConsumer(const std::filesystem::path &path, std::string &consumer);
260
268 void addFileDependency(const std::filesystem::path &path,
269 std::filesystem::path &file_dependency);
270
277 void newFile(const std::filesystem::path &path) const;
278
283 void remove(const std::filesystem::path &path) const;
284
291 void setCommitRule(const std::filesystem::path &path, const std::string &commit_rule);
292
299 void setFireRule(const std::filesystem::path &path, const std::string &fire_rule);
300
306 void setPermanent(const std::filesystem::path &path, bool value);
307
313 void setExclude(const std::filesystem::path &path, bool value);
314
319 void setDirectory(const std::filesystem::path &path);
320
325 void setFile(const std::filesystem::path &path);
326
333 void setCommitedCloseNumber(const std::filesystem::path &path, long num);
334
347
348 void setDirectoryFileCount(const std::filesystem::path &path, long num);
349
356 void setFileDeps(const std::filesystem::path &path,
357 const std::vector<std::filesystem::path> &dependencies);
358
363 void setStoreFileInMemory(const std::filesystem::path &path);
364
368 void setAllStoreInMemory();
369
374 void setStoreFileInFileSystem(const std::filesystem::path &path);
375
380 void setWorkflowName(const std::string &name);
381
387 long getDirectoryFileCount(const std::filesystem::path &path) const;
388
390 std::string getCommitRule(const std::filesystem::path &path) const;
391
393 std::string getFireRule(const std::filesystem::path &path) const;
394
396 std::vector<std::string> getProducers(const std::filesystem::path &path) const;
397
399 std::vector<std::string> getConsumers(const std::filesystem::path &path) const;
400
402 long getCommitCloseCount(const std::filesystem::path &path) const;
403
405 std::vector<std::filesystem::path>
406 getCommitOnFileDependencies(const std::filesystem::path &path) const;
407
409 std::vector<std::string> getFileToStoreInMemory() const;
410
412 std::set<std::string> getHomeNode(const std::filesystem::path &path) const;
413
415 void setHomeNode(const std::filesystem::path &path) const;
416
418 const std::string &getWorkflowName() const;
419
426 bool isProducer(const std::filesystem::path &path, const std::string &app_name) const;
427
434 bool isConsumer(const std::filesystem::path &path, const std::string &app_name) const;
435
441 bool isFirable(const std::filesystem::path &path) const;
442
448 bool isFile(const std::filesystem::path &path) const;
449
455 bool isExcluded(const std::filesystem::path &path) const;
456
462 bool isDirectory(const std::filesystem::path &path) const;
463
469 bool isStoredInMemory(const std::filesystem::path &path) const;
470
476 bool isPermanent(const std::filesystem::path &path) const;
477
483 bool isCommitted(const std::filesystem::path &path) const;
484
492 bool increaseCloseCount(const std::filesystem::path &path) const;
493
498 void setCommitted(const std::filesystem::path &path) const;
499
504 std::vector<std::string> getPaths() const;
505
511 bool operator==(const Engine &other) const;
512
517 void loadConfiguration(const std::string &path);
518
523
528 void startApiServer();
529};
530
531} // namespace capiocl::engine
532
533#endif // CAPIO_CL_ENGINE_H
Load configuration and store it from a CAPIO-CL TOML configuration file.
Definition configuration.h:41
bool isDirectory(const std::filesystem::path &path) const
Check if a path is a directory.
Definition Engine.cpp:818
void setStoreFileInFileSystem(const std::filesystem::path &path)
Store the file on the file system.
Definition Engine.cpp:1079
void setDirectory(const std::filesystem::path &path)
Mark a path as a directory.
Definition Engine.cpp:758
std::set< std::string > getHomeNode(const std::filesystem::path &path) const
Get the home node of a file.
Definition Engine.cpp:1134
bool isStoredInMemory(const std::filesystem::path &path) const
Check if a file is stored in memory.
Definition Engine.cpp:1098
void setCommitted(const std::filesystem::path &path) const
Definition Engine.cpp:721
const std::string & getWorkflowName() const
Get current workflow name loaded from memory.
Definition Engine.cpp:1073
bool increaseCloseCount(const std::filesystem::path &path) const
Record one qualified logical close for an ON_CLOSE path. Filesystem-backed counts greater than one re...
Definition Engine.cpp:692
bool contains(const std::filesystem::path &file) const
Check whether a file is contained in the configuration. The lookup is performed by exact match or by ...
Definition Engine.cpp:328
bool isProducer(const std::filesystem::path &path, const std::string &app_name) const
Check if a process is a producer for a file.
Definition Engine.cpp:946
void loadConfiguration(const std::string &path)
Definition Engine.cpp:1183
void setCommitedCloseNumber(const std::filesystem::path &path, long num)
Set the commit-on-close counter. The file will be committed after num close operations.
Definition Engine.cpp:826
std::vector< std::filesystem::path > getCommitOnFileDependencies(const std::filesystem::path &path) const
Get file dependencies.
Definition Engine.cpp:1019
bool isPermanent(const std::filesystem::path &path) const
Check if file should remain on file system after workflow terminates.
Definition Engine.cpp:625
std::vector< std::string > getPaths() const
Definition Engine.cpp:727
void addFileDependency(const std::filesystem::path &path, std::filesystem::path &file_dependency)
Add a new file dependency, when rule is commit_on_file As a side effect, the file identified by path,...
Definition Engine.cpp:467
bool isCommitted(const std::filesystem::path &path) const
Definition Engine.cpp:645
std::vector< std::string > getConsumers(const std::filesystem::path &path) const
Get the consumers of a file.
Definition Engine.cpp:890
void addProducer(const std::filesystem::path &path, std::string &producer)
Add a new producer to a file entry.
Definition Engine.cpp:413
std::string getCommitRule(const std::filesystem::path &path) const
Get the commit rule of a file.
Definition Engine.cpp:517
void setStoreFileInMemory(const std::filesystem::path &path)
Store the file in memory only.
Definition Engine.cpp:1029
size_t size() const
return the number of entries in the current configuration
Definition Engine.cpp:339
void setFireRule(const std::filesystem::path &path, const std::string &fire_rule)
Set the fire rule of a file.
Definition Engine.cpp:559
std::vector< std::string > getProducers(const std::filesystem::path &path) const
Get the producers of a file.
Definition Engine.cpp:925
void newFile(const std::filesystem::path &path) const
Create a new CAPIO file entry. Commit and fire rules are automatically computed using the longest pre...
Definition Engine.cpp:389
std::vector< std::string > getFileToStoreInMemory() const
Get the list of files stored in memory.
Definition Engine.cpp:1118
bool isFile(const std::filesystem::path &path) const
Check if a path refers to a file.
Definition Engine.cpp:798
std::string getFireRule(const std::filesystem::path &path) const
Get the fire rule of a file.
Definition Engine.cpp:538
long getDirectoryFileCount(const std::filesystem::path &path) const
Get the expected number of files in a directory.
Definition Engine.cpp:395
void setDirectoryFileCount(const std::filesystem::path &path, long num)
Sets the expected number of files in a directory.
Definition Engine.cpp:851
void setFile(const std::filesystem::path &path)
Mark a path as a file.
Definition Engine.cpp:778
void remove(const std::filesystem::path &path) const
Remove a file from the configuration.
Definition Engine.cpp:877
void useDefaultConfiguration()
Definition Engine.cpp:1225
void setWorkflowName(const std::string &name)
Definition Engine.cpp:1064
void startApiServer()
Definition Engine.cpp:1235
bool isConsumer(const std::filesystem::path &path, const std::string &app_name) const
Check if a process is a consumer for a file.
Definition Engine.cpp:900
void setPermanent(const std::filesystem::path &path, bool value)
Mark a file as permanent or not.
Definition Engine.cpp:604
bool isExcluded(const std::filesystem::path &path) const
Check if a path is excluded.
Definition Engine.cpp:1145
bool operator==(const Engine &other) const
Check for equality between two instances of Engine.
Definition Engine.cpp:1164
bool isFirable(const std::filesystem::path &path) const
Check if a file is firable, that is fire rule is no_update.
Definition Engine.cpp:584
void addConsumer(const std::filesystem::path &path, std::string &consumer)
Add a new consumer to a file entry.
Definition Engine.cpp:442
long getCommitCloseCount(const std::filesystem::path &path) const
Get the commit-on-close counter for a file.
Definition Engine.cpp:998
void add(std::filesystem::path &path, std::vector< std::string > &producers, std::vector< std::string > &consumers, const std::string &commit_rule, const std::string &fire_rule, bool permanent, bool exclude, std::vector< std::filesystem::path > &dependencies)
Add a new CAPIO-CL configuration entry.
Definition Engine.cpp:345
void print() const
Print the current CAPIO-CL configuration.
Definition Engine.cpp:43
void setAllStoreInMemory()
set all files to be stored in memory. Once this method is called, all new files will be stored in mem...
Definition Engine.cpp:1050
void setCommitRule(const std::filesystem::path &path, const std::string &commit_rule)
Set the commit rule of a file.
Definition Engine.cpp:491
Engine(bool use_default_settings=true)
Class constructor.
Definition Engine.cpp:165
void setFileDeps(const std::filesystem::path &path, const std::vector< std::filesystem::path > &dependencies)
Set the dependencies of a file. This method as a side effect sets the commit rule to Commit on Files.
Definition Engine.cpp:969
void setHomeNode(const std::filesystem::path &path) const
Set the home node for a given path.
Definition Engine.cpp:1139
void setExclude(const std::filesystem::path &path, bool value)
Mark a file as excluded or not.
Definition Engine.cpp:738
Class to monitor runtime dependent information on CAPIO-CL related paths, such as commitment status a...
Definition monitor.h:326
Dump the current loaded CAPIO-CL configuration from class Engine to a CAPIO-CL configuration file.
Definition serializer.h:34
constexpr char ON_TERMINATION[]
CoT Streaming Rule.
Definition capiocl.hpp:45
Namespace containing the CAPIO-CL Engine.
Definition engine.h:13
constexpr char UPDATE[]
FoC Streaming Rule.
Definition capiocl.hpp:18
Internal CAPIO-CL Engine storage entity. Each CapioCLEntry is an entry for a given file handled by CA...
Definition engine.h:17
bool excluded
whether to ignore this entry
Definition engine.h:40
bool is_file
whether this entry is a file or a directory
Definition engine.h:42
bool operator!=(const CapioCLEntry &other)
check for inequality
Definition Engine.cpp:1388
long commit_on_close_count
Expected close count.
Definition engine.h:32
bool enable_directory_count_update
whether to update or not directory item count
Definition engine.h:34
std::vector< std::filesystem::path > file_dependencies
Dependencies for Commit.
Definition engine.h:24
std::string fire_rule
Fire rule.
Definition engine.h:28
CapioCLEntry operator+(const CapioCLEntry &rhs)
add a new CapioClEntry to this one
Definition Engine.cpp:1331
bool permanent
whether the file should persiste after workflow termination
Definition engine.h:38
std::string toJson() const
Serialize this entry to a JSON object returned as string to be sent over network.
Definition Engine.cpp:1279
CapioCLEntry & operator+=(const CapioCLEntry &rhs)
add a new CapioClEntry to this one
Definition Engine.cpp:1304
bool operator==(const CapioCLEntry &other)
check for equality of rules
Definition Engine.cpp:1338
std::string commit_rule
Commit rule.
Definition engine.h:26
static CapioCLEntry fromJson(const std::string &in)
Definition Engine.cpp:1241
long directory_children_count
Expected number of files in directory.
Definition engine.h:30
bool store_in_memory
Store in memory or on the file system.
Definition engine.h:36
std::vector< std::string > consumers
consumers of file
Definition engine.h:22
std::vector< std::string > producers
Producers of file.
Definition engine.h:20