1#ifndef CAPIO_CL_ENGINE_H
2#define CAPIO_CL_ENGINE_H
6#include "capiocl/monitor.h"
7#include "capiocl/serializer.h"
28 bool store_all_in_memory =
false;
37 std::string node_name;
40 std::string workflow_name;
45 struct CapioCLEntry final {
46 std::vector<std::string> producers;
47 std::vector<std::string> consumers;
48 std::vector<std::filesystem::path> file_dependencies;
51 long directory_children_count = 0;
52 long commit_on_close_count = 0;
53 bool enable_directory_count_update =
true;
54 bool store_in_memory =
false;
55 bool permanent =
false;
56 bool excluded =
false;
63 mutable std::unordered_map<std::string, CapioCLEntry> _capio_cl_entries;
75 static std::string truncateLastN(
const std::string &str,
const std::size_t n) {
76 if (str.length() > n) {
77 return "[..] " + str.substr(str.length() - n);
86 void _newFile(
const std::filesystem::path &path)
const;
100 void compute_directory_entry_count(
const std::filesystem::path &path)
const;
104 explicit Engine(
bool use_default_settings =
true);
116 bool contains(
const std::filesystem::path &file)
const;
133 void add(std::filesystem::path &path, std::vector<std::string> &producers,
134 std::vector<std::string> &consumers,
const std::string &commit_rule,
135 const std::string &fire_rule,
bool permanent,
bool exclude,
136 std::vector<std::filesystem::path> &dependencies);
144 void addProducer(
const std::filesystem::path &path, std::string &producer);
152 void addConsumer(
const std::filesystem::path &path, std::string &consumer);
162 std::filesystem::path &file_dependency);
170 void newFile(
const std::filesystem::path &path);
176 void remove(
const std::filesystem::path &path)
const;
184 void setCommitRule(
const std::filesystem::path &path,
const std::string &commit_rule);
192 void setFireRule(
const std::filesystem::path &path,
const std::string &fire_rule);
199 void setPermanent(
const std::filesystem::path &path,
bool value);
206 void setExclude(
const std::filesystem::path &path,
bool value);
218 void setFile(
const std::filesystem::path &path);
249 void setFileDeps(
const std::filesystem::path &path,
250 const std::vector<std::filesystem::path> &dependencies);
283 std::string
getCommitRule(
const std::filesystem::path &path)
const;
286 std::string
getFireRule(
const std::filesystem::path &path)
const;
289 std::vector<std::string>
getProducers(
const std::filesystem::path &path)
const;
292 std::vector<std::string>
getConsumers(
const std::filesystem::path &path)
const;
298 std::vector<std::filesystem::path>
305 std::set<std::string>
getHomeNode(
const std::filesystem::path &path)
const;
308 void setHomeNode(
const std::filesystem::path &path)
const;
319 bool isProducer(
const std::filesystem::path &path,
const std::string &app_name)
const;
327 bool isConsumer(
const std::filesystem::path &path,
const std::string &app_name)
const;
334 bool isFirable(
const std::filesystem::path &path)
const;
341 bool isFile(
const std::filesystem::path &path)
const;
348 bool isExcluded(
const std::filesystem::path &path)
const;
355 bool isDirectory(
const std::filesystem::path &path)
const;
369 bool isPermanent(
const std::filesystem::path &path)
const;
376 bool isCommitted(
const std::filesystem::path &path)
const;
382 void setCommitted(
const std::filesystem::path &path)
const;
388 std::vector<std::string>
getPaths()
const;
Load configuration and store it from a CAPIO-CL TOML configuration file.
Definition configuration.h:31
bool isDirectory(const std::filesystem::path &path) const
Check if a path is a directory.
Definition Engine.cpp:467
void setStoreFileInFileSystem(const std::filesystem::path &path)
Store the file on the file system.
Definition Engine.cpp:641
void setDirectory(const std::filesystem::path &path)
Mark a path as a directory.
Definition Engine.cpp:429
std::set< std::string > getHomeNode(const std::filesystem::path &path) const
Get the home node of a file.
Definition Engine.cpp:680
bool isStoredInMemory(const std::filesystem::path &path) const
Check if a file is stored in memory.
Definition Engine.cpp:654
void setCommitted(const std::filesystem::path &path) const
Definition Engine.cpp:404
const std::string & getWorkflowName() const
Get current workflow name loaded from memory.
Definition Engine.cpp:639
bool contains(const std::filesystem::path &file) const
Check whether a file is contained in the configuration. The lookup is performed by exact match or by ...
Definition Engine.cpp:203
bool isProducer(const std::filesystem::path &path, const std::string &app_name) const
Check if a process is a producer for a file.
Definition Engine.cpp:552
void loadConfiguration(const std::string &path)
Definition Engine.cpp:761
void setCommitedCloseNumber(const std::filesystem::path &path, long num)
Set the commit-on-close counter. The file will be committed after num close operations.
Definition Engine.cpp:475
std::vector< std::filesystem::path > getCommitOnFileDependencies(const std::filesystem::path &path) const
Get file dependencies.
Definition Engine.cpp:608
bool isPermanent(const std::filesystem::path &path) const
Check if file should remain on file system after workflow terminates.
Definition Engine.cpp:387
long getCommitCloseCount(const std::filesystem::path::iterator::reference &path) const
Get the commit-on-close counter for a file.
Definition Engine.cpp:593
std::vector< std::string > getPaths() const
Definition Engine.cpp:408
void addFileDependency(const std::filesystem::path &path, std::filesystem::path &file_dependency)
Add a new file dependency, when rule is commit_on_file As a side effect, the file identified by path,...
Definition Engine.cpp:285
bool isCommitted(const std::filesystem::path &path) const
Definition Engine.cpp:400
std::vector< std::string > getConsumers(const std::filesystem::path &path) const
Get the consumers of a file.
Definition Engine.cpp:513
void addProducer(const std::filesystem::path &path, std::string &producer)
Add a new producer to a file entry.
Definition Engine.cpp:246
std::string getCommitRule(const std::filesystem::path &path) const
Get the commit rule of a file.
Definition Engine.cpp:319
void setStoreFileInMemory(const std::filesystem::path &path)
Store the file in memory only.
Definition Engine.cpp:615
size_t size() const
return the number of entries in the current configuration
Definition Engine.cpp:209
void setFireRule(const std::filesystem::path &path, const std::string &fire_rule)
Set the fire rule of a file.
Definition Engine.cpp:345
std::vector< std::string > getProducers(const std::filesystem::path &path) const
Get the producers of a file.
Definition Engine.cpp:540
std::vector< std::string > getFileToStoreInMemory() const
Get the list of files stored in memory.
Definition Engine.cpp:667
bool isFile(const std::filesystem::path &path) const
Check if a path refers to a file.
Definition Engine.cpp:455
std::string getFireRule(const std::filesystem::path &path) const
Get the fire rule of a file.
Definition Engine.cpp:332
long getDirectoryFileCount(const std::filesystem::path &path) const
Get the expected number of files in a directory.
Definition Engine.cpp:234
void setDirectoryFileCount(const std::filesystem::path &path, long num)
Sets the expected number of files in a directory.
Definition Engine.cpp:489
void newFile(const std::filesystem::path &path)
Create a new CAPIO file entry. Commit and fire rules are automatically computed using the longest pre...
Definition Engine.cpp:232
void setFile(const std::filesystem::path &path)
Mark a path as a file.
Definition Engine.cpp:442
void remove(const std::filesystem::path &path) const
Remove a file from the configuration.
Definition Engine.cpp:505
void useDefaultConfiguration()
Definition Engine.cpp:767
void setWorkflowName(const std::string &name)
Definition Engine.cpp:635
bool isConsumer(const std::filesystem::path &path, const std::string &app_name) const
Check if a process is a consumer for a file.
Definition Engine.cpp:520
void setPermanent(const std::filesystem::path &path, bool value)
Mark a file as permanent or not.
Definition Engine.cpp:374
bool isExcluded(const std::filesystem::path &path) const
Check if a path is excluded.
Definition Engine.cpp:688
bool operator==(const Engine &other) const
Check for equality between two instances of Engine.
Definition Engine.cpp:701
bool isFirable(const std::filesystem::path &path) const
Check if a file is firable, that is fire rule is no_update.
Definition Engine.cpp:361
void addConsumer(const std::filesystem::path &path, std::string &consumer)
Add a new consumer to a file entry.
Definition Engine.cpp:267
void add(std::filesystem::path &path, std::vector< std::string > &producers, std::vector< std::string > &consumers, const std::string &commit_rule, const std::string &fire_rule, bool permanent, bool exclude, std::vector< std::filesystem::path > &dependencies)
Add a new CAPIO-CL configuration entry.
Definition Engine.cpp:211
void print() const
Print the current CAPIO-CL configuration.
Definition Engine.cpp:10
void setAllStoreInMemory()
set all files to be stored in memory. Once this method is called, all new files will be stored in mem...
Definition Engine.cpp:628
void setCommitRule(const std::filesystem::path &path, const std::string &commit_rule)
Set the commit rule of a file.
Definition Engine.cpp:303
Engine(bool use_default_settings=true)
Class constructor.
Definition Engine.cpp:129
void setFileDeps(const std::filesystem::path &path, const std::vector< std::filesystem::path > &dependencies)
Set the dependencies of a file. This method as a side effect sets the commit rule to Commit on Files.
Definition Engine.cpp:571
void setHomeNode(const std::filesystem::path &path) const
Set the home node for a given path.
Definition Engine.cpp:684
void setExclude(const std::filesystem::path &path, bool value)
Mark a file as excluded or not.
Definition Engine.cpp:416
Class to monitor runtime dependent information on CAPIO-CL related paths, such as commitment status a...
Definition monitor.h:289
Dump the current loaded CAPIO-CL configuration from class Engine to a CAPIO-CL configuration file.
Definition serializer.h:30
constexpr char ON_TERMINATION[]
CoT Streaming Rule.
Definition capiocl.hpp:45
Namespace containing the CAPIO-CL Engine.
Definition engine.h:10
constexpr char UPDATE[]
FoC Streaming Rule.
Definition capiocl.hpp:18