CAPIO-CL 1.3.4
CAPIO-CL: Cross Application Programmable I/O - Coordination Language
Loading...
Searching...
No Matches
engine.h
1#ifndef CAPIO_CL_ENGINE_H
2#define CAPIO_CL_ENGINE_H
3#include <vector>
4
5#include "capiocl.hpp"
6#include "capiocl/monitor.h"
7#include "capiocl/serializer.h"
8#include "capiocl/webapi.h"
9
11namespace capiocl::engine {
27class Engine final {
28 friend class serializer::Serializer;
29 bool store_all_in_memory = false;
30
33
35 monitor::Monitor monitor;
36
38 std::string node_name;
39
41 std::string workflow_name;
42
44 std::unique_ptr<webapi::CapioClWebApiServer> webapi_server;
45
46 // LCOV_EXCL_START
49 struct CapioCLEntry final {
50 std::vector<std::string> producers;
51 std::vector<std::string> consumers;
52 std::vector<std::filesystem::path> file_dependencies;
53 std::string commit_rule = commitRules::ON_TERMINATION;
54 std::string fire_rule = fireRules::UPDATE;
55 long directory_children_count = 0;
56 long commit_on_close_count = 0;
57 bool enable_directory_count_update = true; // whether to update or not directory item count
58 bool store_in_memory = false;
59 bool permanent = false;
60 bool excluded = false;
61 bool is_file = true;
62 };
63
64 // LCOV_EXCL_STOP
65
67 mutable std::unordered_map<std::string, CapioCLEntry> _capio_cl_entries;
68
79 static std::string truncateLastN(const std::string &str, const std::size_t n) {
80 if (str.length() > n) {
81 return "[..] " + str.substr(str.length() - n);
82 }
83 return str;
84 }
85
90 void _newFile(const std::filesystem::path &path) const;
91
104 void compute_directory_entry_count(const std::filesystem::path &path) const;
105
106 public:
108 explicit Engine(bool use_default_settings = true);
109
111 void print() const;
112
120 bool contains(const std::filesystem::path &file) const;
121
123 size_t size() const;
124
137 void add(std::filesystem::path &path, std::vector<std::string> &producers,
138 std::vector<std::string> &consumers, const std::string &commit_rule,
139 const std::string &fire_rule, bool permanent, bool exclude,
140 std::vector<std::filesystem::path> &dependencies);
141
148 void addProducer(const std::filesystem::path &path, std::string &producer);
149
156 void addConsumer(const std::filesystem::path &path, std::string &consumer);
157
165 void addFileDependency(const std::filesystem::path &path,
166 std::filesystem::path &file_dependency);
167
174 void newFile(const std::filesystem::path &path);
175
180 void remove(const std::filesystem::path &path) const;
181
188 void setCommitRule(const std::filesystem::path &path, const std::string &commit_rule);
189
196 void setFireRule(const std::filesystem::path &path, const std::string &fire_rule);
197
203 void setPermanent(const std::filesystem::path &path, bool value);
204
210 void setExclude(const std::filesystem::path &path, bool value);
211
216 void setDirectory(const std::filesystem::path &path);
217
222 void setFile(const std::filesystem::path &path);
223
230 void setCommitedCloseNumber(const std::filesystem::path &path, long num);
231
244
245 void setDirectoryFileCount(const std::filesystem::path &path, long num);
246
253 void setFileDeps(const std::filesystem::path &path,
254 const std::vector<std::filesystem::path> &dependencies);
255
260 void setStoreFileInMemory(const std::filesystem::path &path);
261
265 void setAllStoreInMemory();
266
271 void setStoreFileInFileSystem(const std::filesystem::path &path);
272
277 void setWorkflowName(const std::string &name);
278
284 long getDirectoryFileCount(const std::filesystem::path &path) const;
285
287 std::string getCommitRule(const std::filesystem::path &path) const;
288
290 std::string getFireRule(const std::filesystem::path &path) const;
291
293 std::vector<std::string> getProducers(const std::filesystem::path &path) const;
294
296 std::vector<std::string> getConsumers(const std::filesystem::path &path) const;
297
299 long getCommitCloseCount(const std::filesystem::path::iterator::reference &path) const;
300
302 std::vector<std::filesystem::path>
303 getCommitOnFileDependencies(const std::filesystem::path &path) const;
304
306 std::vector<std::string> getFileToStoreInMemory() const;
307
309 std::set<std::string> getHomeNode(const std::filesystem::path &path) const;
310
312 void setHomeNode(const std::filesystem::path &path) const;
313
315 const std::string &getWorkflowName() const;
316
323 bool isProducer(const std::filesystem::path &path, const std::string &app_name) const;
324
331 bool isConsumer(const std::filesystem::path &path, const std::string &app_name) const;
332
338 bool isFirable(const std::filesystem::path &path) const;
339
345 bool isFile(const std::filesystem::path &path) const;
346
352 bool isExcluded(const std::filesystem::path &path) const;
353
359 bool isDirectory(const std::filesystem::path &path) const;
360
366 bool isStoredInMemory(const std::filesystem::path &path) const;
367
373 bool isPermanent(const std::filesystem::path &path) const;
374
380 bool isCommitted(const std::filesystem::path &path) const;
381
386 void setCommitted(const std::filesystem::path &path) const;
387
392 std::vector<std::string> getPaths() const;
393
399 bool operator==(const Engine &other) const;
400
405 void loadConfiguration(const std::string &path);
406
411
417 void startApiServer(const std::string &address = "127.0.0.1", int port = 5520);
418};
419
420} // namespace capiocl::engine
421
422#endif // CAPIO_CL_ENGINE_H
Load configuration and store it from a CAPIO-CL TOML configuration file.
Definition configuration.h:31
bool isDirectory(const std::filesystem::path &path) const
Check if a path is a directory.
Definition Engine.cpp:468
void setStoreFileInFileSystem(const std::filesystem::path &path)
Store the file on the file system.
Definition Engine.cpp:642
void setDirectory(const std::filesystem::path &path)
Mark a path as a directory.
Definition Engine.cpp:430
std::set< std::string > getHomeNode(const std::filesystem::path &path) const
Get the home node of a file.
Definition Engine.cpp:681
bool isStoredInMemory(const std::filesystem::path &path) const
Check if a file is stored in memory.
Definition Engine.cpp:655
void setCommitted(const std::filesystem::path &path) const
Definition Engine.cpp:405
const std::string & getWorkflowName() const
Get current workflow name loaded from memory.
Definition Engine.cpp:640
bool contains(const std::filesystem::path &file) const
Check whether a file is contained in the configuration. The lookup is performed by exact match or by ...
Definition Engine.cpp:204
bool isProducer(const std::filesystem::path &path, const std::string &app_name) const
Check if a process is a producer for a file.
Definition Engine.cpp:553
void loadConfiguration(const std::string &path)
Definition Engine.cpp:762
void setCommitedCloseNumber(const std::filesystem::path &path, long num)
Set the commit-on-close counter. The file will be committed after num close operations.
Definition Engine.cpp:476
std::vector< std::filesystem::path > getCommitOnFileDependencies(const std::filesystem::path &path) const
Get file dependencies.
Definition Engine.cpp:609
bool isPermanent(const std::filesystem::path &path) const
Check if file should remain on file system after workflow terminates.
Definition Engine.cpp:388
long getCommitCloseCount(const std::filesystem::path::iterator::reference &path) const
Get the commit-on-close counter for a file.
Definition Engine.cpp:594
std::vector< std::string > getPaths() const
Definition Engine.cpp:409
void addFileDependency(const std::filesystem::path &path, std::filesystem::path &file_dependency)
Add a new file dependency, when rule is commit_on_file As a side effect, the file identified by path,...
Definition Engine.cpp:286
bool isCommitted(const std::filesystem::path &path) const
Definition Engine.cpp:401
std::vector< std::string > getConsumers(const std::filesystem::path &path) const
Get the consumers of a file.
Definition Engine.cpp:514
void addProducer(const std::filesystem::path &path, std::string &producer)
Add a new producer to a file entry.
Definition Engine.cpp:247
void startApiServer(const std::string &address="127.0.0.1", int port=5520)
Definition Engine.cpp:776
std::string getCommitRule(const std::filesystem::path &path) const
Get the commit rule of a file.
Definition Engine.cpp:320
void setStoreFileInMemory(const std::filesystem::path &path)
Store the file in memory only.
Definition Engine.cpp:616
size_t size() const
return the number of entries in the current configuration
Definition Engine.cpp:210
void setFireRule(const std::filesystem::path &path, const std::string &fire_rule)
Set the fire rule of a file.
Definition Engine.cpp:346
std::vector< std::string > getProducers(const std::filesystem::path &path) const
Get the producers of a file.
Definition Engine.cpp:541
std::vector< std::string > getFileToStoreInMemory() const
Get the list of files stored in memory.
Definition Engine.cpp:668
bool isFile(const std::filesystem::path &path) const
Check if a path refers to a file.
Definition Engine.cpp:456
std::string getFireRule(const std::filesystem::path &path) const
Get the fire rule of a file.
Definition Engine.cpp:333
long getDirectoryFileCount(const std::filesystem::path &path) const
Get the expected number of files in a directory.
Definition Engine.cpp:235
void setDirectoryFileCount(const std::filesystem::path &path, long num)
Sets the expected number of files in a directory.
Definition Engine.cpp:490
void newFile(const std::filesystem::path &path)
Create a new CAPIO file entry. Commit and fire rules are automatically computed using the longest pre...
Definition Engine.cpp:233
void setFile(const std::filesystem::path &path)
Mark a path as a file.
Definition Engine.cpp:443
void remove(const std::filesystem::path &path) const
Remove a file from the configuration.
Definition Engine.cpp:506
void useDefaultConfiguration()
Definition Engine.cpp:768
void setWorkflowName(const std::string &name)
Definition Engine.cpp:636
bool isConsumer(const std::filesystem::path &path, const std::string &app_name) const
Check if a process is a consumer for a file.
Definition Engine.cpp:521
void setPermanent(const std::filesystem::path &path, bool value)
Mark a file as permanent or not.
Definition Engine.cpp:375
bool isExcluded(const std::filesystem::path &path) const
Check if a path is excluded.
Definition Engine.cpp:689
bool operator==(const Engine &other) const
Check for equality between two instances of Engine.
Definition Engine.cpp:702
bool isFirable(const std::filesystem::path &path) const
Check if a file is firable, that is fire rule is no_update.
Definition Engine.cpp:362
void addConsumer(const std::filesystem::path &path, std::string &consumer)
Add a new consumer to a file entry.
Definition Engine.cpp:268
void add(std::filesystem::path &path, std::vector< std::string > &producers, std::vector< std::string > &consumers, const std::string &commit_rule, const std::string &fire_rule, bool permanent, bool exclude, std::vector< std::filesystem::path > &dependencies)
Add a new CAPIO-CL configuration entry.
Definition Engine.cpp:212
void print() const
Print the current CAPIO-CL configuration.
Definition Engine.cpp:11
void setAllStoreInMemory()
set all files to be stored in memory. Once this method is called, all new files will be stored in mem...
Definition Engine.cpp:629
void setCommitRule(const std::filesystem::path &path, const std::string &commit_rule)
Set the commit rule of a file.
Definition Engine.cpp:304
Engine(bool use_default_settings=true)
Class constructor.
Definition Engine.cpp:130
void setFileDeps(const std::filesystem::path &path, const std::vector< std::filesystem::path > &dependencies)
Set the dependencies of a file. This method as a side effect sets the commit rule to Commit on Files.
Definition Engine.cpp:572
void setHomeNode(const std::filesystem::path &path) const
Set the home node for a given path.
Definition Engine.cpp:685
void setExclude(const std::filesystem::path &path, bool value)
Mark a file as excluded or not.
Definition Engine.cpp:417
Class to monitor runtime dependent information on CAPIO-CL related paths, such as commitment status a...
Definition monitor.h:295
Dump the current loaded CAPIO-CL configuration from class Engine to a CAPIO-CL configuration file.
Definition serializer.h:30
constexpr char ON_TERMINATION[]
CoT Streaming Rule.
Definition capiocl.hpp:45
Namespace containing the CAPIO-CL Engine.
Definition engine.h:11
constexpr char UPDATE[]
FoC Streaming Rule.
Definition capiocl.hpp:18