diff --git a/arcane/src/arcane/impl/VariableIOReaderMng.cc b/arcane/src/arcane/impl/VariableIOReaderMng.cc index 04e9094f8..32bb06eb5 100644 --- a/arcane/src/arcane/impl/VariableIOReaderMng.cc +++ b/arcane/src/arcane/impl/VariableIOReaderMng.cc @@ -21,6 +21,8 @@ #include "arcane/utils/MD5HashAlgorithm.h" #include "arcane/utils/OStringStream.h" #include "arcane/utils/PlatformUtils.h" +#include "arcane/utils/JSONReader.h" +#include "arcane/utils/ValueConvert.h" #include "arcane/core/IXmlDocumentHolder.h" #include "arcane/core/XmlNode.h" @@ -367,7 +369,10 @@ VariableIOReaderMng:: VariableIOReaderMng(VariableMng* vm) : TraceAccessor(vm->traceMng()) , m_variable_mng(vm) +, m_is_use_json_metadata(false) { + if (auto v = Convert::Type::tryParseFromEnvironment("ARCANE_USE_JSON_METADATA", true)) + m_is_use_json_metadata = (v.value()!=0); } /*---------------------------------------------------------------------------*/ @@ -482,12 +487,33 @@ _readMetaData(VariableMetaDataList& vmd_list, Span bytes) ScopedPtrT doc(IXmlDocumentHolder::loadFromBuffer(bytes, "meta_data", traceMng())); if (!doc.get()) ARCANE_FATAL("The meta-data are invalid"); + JSONDocument json_reader; XmlNode root_node = doc->documentNode().documentElement(); + XmlNode json_node = root_node.child("json"); + + // A partir de la version 3.11 de Arcane (juillet 2023), les + // méta-données sont aussi disponibles au format JSON. On les utilise + // si 'm_is_use_json_metadata' est vrai. + JSONValue json_variables; + JSONValue json_meshes; + if (m_is_use_json_metadata && !json_node.null()) { + String json_meta_data = json_node.value(); + info(6) << "READER_JSON=" << json_meta_data; + json_reader.parse(json_meta_data.bytes()); + //json_reader_ptr = &json_reader; + JSONValue json_meta_data_object = json_reader.root().expectedChild("arcane-checkpoint-metadata"); + JSONValue json_version = json_meta_data_object.expectedChild("version"); + Int32 v = json_version.valueAsInt32(); + if (v != 1) + ARCANE_FATAL("Bad version for JSON Meta Data (v={0}). Only version '1' is supported", v); + json_variables = json_meta_data_object.expectedChild("variables"); + json_meshes = json_meta_data_object.expectedChild("meshes"); + } XmlNode variables_node = root_node.child("variables"); - _readVariablesMetaData(vmd_list, variables_node); + _readVariablesMetaData(vmd_list, json_variables, variables_node); XmlNode meshes_node = root_node.child("meshes"); - _readMeshesMetaData(meshes_node); + _readMeshesMetaData(json_meshes, meshes_node); } /*---------------------------------------------------------------------------*/ @@ -584,9 +610,8 @@ _createVariablesFromMetaData(const VariableMetaDataList& vmd_list) /*---------------------------------------------------------------------------*/ void VariableIOReaderMng:: -_readVariablesMetaData(VariableMetaDataList& vmd_list, const XmlNode& variables_node) +_readVariablesMetaData(VariableMetaDataList& vmd_list, JSONValue variables_json, const XmlNode& variables_node) { - XmlNodeList vars = variables_node.children("variable"); String ustr_base_name("base-name"); String ustr_family_name("item-family-name"); String ustr_group_name("item-group-name"); @@ -597,16 +622,63 @@ _readVariablesMetaData(VariableMetaDataList& vmd_list, const XmlNode& variables_ String ustr_multitag("multi-tag"); vmd_list.clear(); - for (const auto& var : vars) { - String full_type = var.attrValue(ustr_full_type); + struct VariableReadInfo + { + String full_type; + String base_name; + String mesh_name; + String family_name; + String group_name; + String hash_value; + String multi_tag; + Int32 property; + }; + UniqueArray variables_info; + + // Lit les informations des variables à partir des données JSON + // si ces dernières existent. + if (!variables_json.null()) { + // Lecture via JSON + // Déclare la liste ici pour éviter de retourner un temporaire dans 'for-range' + JSONValueList vars = variables_json.valueAsArray(); + for (const JSONValue& var : vars) { + VariableReadInfo r; + r.full_type = var.expectedChild(ustr_full_type).value(); + r.base_name = var.expectedChild(ustr_base_name).value(); + r.mesh_name = var.child(ustr_mesh_name).value(); + r.family_name = var.child(ustr_family_name).value(); + r.group_name = var.child(ustr_group_name).value(); + r.hash_value = var.child(ustr_hash).value(); + r.multi_tag = var.child(ustr_multitag).value(); + r.property = var.child(ustr_property).valueAsInt32(); + variables_info.add(r); + } + } + else { + // Lecture via les données XML + XmlNodeList vars = variables_node.children("variable"); + for (const auto& var : vars) { + VariableReadInfo r; + r.full_type = var.attrValue(ustr_full_type); + r.base_name = var.attrValue(ustr_base_name); + r.mesh_name = var.attrValue(ustr_mesh_name); + r.group_name = var.attrValue(ustr_group_name); + r.family_name = var.attrValue(ustr_family_name); + r.hash_value = var.attrValue(ustr_hash); + r.multi_tag = var.attrValue(ustr_multitag); + r.property = var.attr(ustr_property).valueAsInteger(); + variables_info.add(r); + } + } + + for (const VariableReadInfo& r : variables_info) { + String full_type = r.full_type; VariableDataTypeInfo vdti(full_type); - String base_name = var.attrValue(ustr_base_name); - String mesh_name = var.attrValue(ustr_mesh_name); - String family_name = var.attrValue(ustr_family_name); + String family_name = r.family_name; // Actuellement, si la variable n'est pas partielle alors son groupe // n'est pas sauvé dans les meta-données. Il faut donc le générer. - String group_name = var.attrValue(ustr_group_name); + String group_name = r.group_name; bool is_partial = vdti.isPartial(); if (!is_partial) { // NOTE: Cette construction doit être cohérente avec celle de @@ -614,12 +686,12 @@ _readVariablesMetaData(VariableMetaDataList& vmd_list, const XmlNode& variables_ // dans les meta-données. group_name = "All" + family_name + "s"; } - auto vmd = vmd_list.add(base_name, mesh_name, family_name, group_name, is_partial); + auto vmd = vmd_list.add(r.base_name, r.mesh_name, r.family_name, group_name, is_partial); - vmd->setFullType(var.attrValue(ustr_full_type)); - vmd->setHash(var.attrValue(ustr_hash)); - vmd->setMultiTag(var.attrValue(ustr_multitag)); - vmd->setProperty(var.attr(ustr_property).valueAsInteger()); + vmd->setFullType(full_type); + vmd->setHash(r.hash_value); + vmd->setMultiTag(r.multi_tag); + vmd->setProperty(r.property); info(5) << "CHECK VAR: " << " base-name=" << vmd->baseName() @@ -637,24 +709,57 @@ _readVariablesMetaData(VariableMetaDataList& vmd_list, const XmlNode& variables_ /*---------------------------------------------------------------------------*/ void VariableIOReaderMng:: -_readMeshesMetaData(const XmlNode& meshes_node) +_readMeshesMetaData(JSONValue meshes_json, const XmlNode& meshes_node) { - XmlNodeList meshes = meshes_node.children("mesh"); ISubDomain* sd = m_variable_mng->subDomain(); IMeshMng* mesh_mng = sd->meshMng(); IMeshFactoryMng* mesh_factory_mng = mesh_mng->meshFactoryMng(); - for (XmlNode var : meshes) { - String mesh_name = var.attrValue("name"); - String mesh_factory_name = var.attrValue("factory-name"); + + struct MeshInfo + { + String name; + String factory_name; + bool is_sequential; + }; + UniqueArray meshes_info; + + // Lit les informations des maillages à partir des données JSON + // si ces dernières existent. + if (!meshes_json.null()) { + // Déclare la liste ici pour éviter de retourner un temporaire dans 'for-range' + JSONValueList vars = meshes_json.valueAsArray(); + for (const JSONValue& var : vars) { + String mesh_name = var.expectedChild("name").value(); + String mesh_factory_name = var.child("factory-name").value(); + bool is_sequential = false; + JSONValue v = var.child("sequential"); + if (!v.null()) + is_sequential = v.valueAsBool(); + meshes_info.add({ mesh_name, mesh_factory_name, is_sequential }); + } + } + else { + XmlNodeList meshes = meshes_node.children("mesh"); + for (XmlNode var : meshes) { + String mesh_name = var.attrValue("name"); + String mesh_factory_name = var.attrValue("factory-name"); + bool is_sequential = var.attr("sequential", false).valueAsBoolean(); + meshes_info.add({ mesh_name, mesh_factory_name, is_sequential }); + } + } + + for (const MeshInfo& mesh_info : meshes_info) { + String mesh_name = mesh_info.name; + String mesh_factory_name = mesh_info.factory_name; MeshHandle* mesh_handle = mesh_mng->findMeshHandle(mesh_name, false); IMesh* mesh = (mesh_handle) ? mesh_handle->mesh() : nullptr; if (mesh) continue; - bool is_sequential = var.attr("sequential", false).valueAsBoolean(); + bool is_sequential = mesh_info.is_sequential; info() << "Creating from checkpoint mesh='" << mesh_name << "' sequential?=" << is_sequential << " factory=" << mesh_factory_name; - // Depuis avril 2020, l'attribut 'factory-name' est doit être présent + // Depuis avril 2020, l'attribut 'factory-name' doit être présent // et sa valeur non nulle. if (mesh_factory_name.null()) ARCANE_FATAL("No attribute 'factory-name' for mesh"); diff --git a/arcane/src/arcane/impl/VariableIOWriterMng.cc b/arcane/src/arcane/impl/VariableIOWriterMng.cc index 522859546..8a3d09cdd 100644 --- a/arcane/src/arcane/impl/VariableIOWriterMng.cc +++ b/arcane/src/arcane/impl/VariableIOWriterMng.cc @@ -18,6 +18,7 @@ #include "arcane/utils/ScopedPtr.h" #include "arcane/utils/StringBuilder.h" #include "arcane/utils/MD5HashAlgorithm.h" +#include "arcane/utils/JSONWriter.h" #include "arcane/core/IDataWriter.h" #include "arcane/core/IXmlDocumentHolder.h" @@ -64,9 +65,9 @@ writeCheckpoint(ICheckpointWriter* service) if (!service) ARCANE_FATAL("No protection service specified"); - Trace::Setter mci(traceMng(),_msgClassName()); + Trace::Setter mci(traceMng(), _msgClassName()); - Timer::Phase tp(m_variable_mng->m_time_stats,TP_InputOutput); + Timer::Phase tp(m_variable_mng->m_time_stats, TP_InputOutput); CheckpointSaveFilter save_filter; @@ -74,7 +75,7 @@ writeCheckpoint(ICheckpointWriter* service) IDataWriter* data_writer = service->dataWriter(); if (!data_writer) ARCANE_FATAL("no writer() nor dataWriter()"); - writeVariables(data_writer,&save_filter,true); + writeVariables(data_writer, &save_filter, true); service->notifyEndWrite(); } @@ -84,23 +85,22 @@ writeCheckpoint(ICheckpointWriter* service) void VariableIOWriterMng:: writePostProcessing(IPostProcessorWriter* post_processor) { - Trace::Setter mci(traceMng(),_msgClassName()); + Trace::Setter mci(traceMng(), _msgClassName()); if (!post_processor) ARCANE_FATAL("No post-processing service specified"); - Timer::Phase tp(m_variable_mng->m_time_stats,TP_InputOutput); + Timer::Phase tp(m_variable_mng->m_time_stats, TP_InputOutput); post_processor->notifyBeginWrite(); VariableCollection variables(post_processor->variables()); IDataWriter* data_writer = post_processor->dataWriter(); if (!data_writer) ARCANE_FATAL("no writer() nor dataWriter()"); - writeVariables(data_writer,variables,false); + writeVariables(data_writer, variables, false); post_processor->notifyEndWrite(); } - /*---------------------------------------------------------------------------*/ /*---------------------------------------------------------------------------*/ @@ -149,41 +149,67 @@ writeVariables(IDataWriter* writer, const VariableCollection& vars, bool use_has /*---------------------------------------------------------------------------*/ /*---------------------------------------------------------------------------*/ +namespace +{ + void _writeAttribute(JSONWriter& json_writer, XmlNode var_node, const String& name, const String& value) + { + var_node.setAttrValue(name, value); + json_writer.write(name, value); + } + void _writeAttribute(JSONWriter& json_writer, XmlNode var_node, const String& name, Int32 value) + { + Int64 v = value; + var_node.setAttrValue(name, String::fromNumber(v)); + json_writer.write(name, v); + } + void _writeAttribute(JSONWriter& json_writer, XmlNode var_node, const String& name, bool value) + { + var_node.setAttrValue(name, String::fromNumber(value)); + json_writer.write(name, value); + } + +} // namespace void VariableIOWriterMng:: -_generateVariablesMetaData(XmlNode variables_node, const VariableCollection& vars, bool use_hash) +_generateVariablesMetaData(JSONWriter& json_writer, XmlNode variables_node, + const VariableCollection& vars, bool use_hash) { StringBuilder var_full_type_b; ByteUniqueArray hash_values; MD5HashAlgorithm hash_algo; + json_writer.writeKey("variables"); + json_writer.beginArray(); + for (VariableCollection::Enumerator i(vars); ++i;) { + JSONWriter::Object o(json_writer); IVariable* var = *i; ScopedPtrT vmd(var->createMetaData()); String var_full_type = vmd->fullType(); String var_family_name = var->itemFamilyName(); String var_mesh_name = var->meshName(); XmlNode var_node = XmlElement(variables_node, "variable"); - var_node.setAttrValue("base-name", var->name()); + _writeAttribute(json_writer, var_node, "base-name", var->name()); if (!var_family_name.null()) - var_node.setAttrValue("item-family-name", var_family_name); + _writeAttribute(json_writer, var_node, "item-family-name", var_family_name); if (var->isPartial()) - var_node.setAttrValue("item-group-name", var->itemGroupName()); - if (!var_mesh_name.null()) { - var_node.setAttrValue("mesh-name", var_mesh_name); - } - var_node.setAttrValue("full-type", var_full_type); - var_node.setAttrValue("data-type", dataTypeName(var->dataType())); - var_node.setAttrValue("dimension", String::fromNumber(var->dimension())); - var_node.setAttrValue("multi-tag", String::fromNumber(var->multiTag())); - var_node.setAttrValue("property", String::fromNumber(var->property())); + _writeAttribute(json_writer, var_node, "item-group-name", var->itemGroupName()); + if (!var_mesh_name.null()) + _writeAttribute(json_writer, var_node, "mesh-name", var_mesh_name); + _writeAttribute(json_writer, var_node, "full-type", var_full_type); + _writeAttribute(json_writer, var_node, "data-type", dataTypeName(var->dataType())); + _writeAttribute(json_writer, var_node, "dimension", var->dimension()); + _writeAttribute(json_writer, var_node, "multi-tag", var->multiTag()); + _writeAttribute(json_writer, var_node, "property", var->property()); if (use_hash) { hash_values.clear(); var->data()->computeHash(&hash_algo, hash_values); String hash_str = Convert::toHexaString(hash_values); - var_node.setAttrValue("hash", hash_str); + _writeAttribute(json_writer, var_node, "hash", hash_str); } } + + json_writer.endArray(); } /*---------------------------------------------------------------------------*/ @@ -193,10 +219,14 @@ _generateVariablesMetaData(XmlNode variables_node, const VariableCollection& var * TODO Cela doit normalement être fait via le 'IMeshMng'. */ void VariableIOWriterMng:: -_generateMeshesMetaData(XmlNode meshes_node) +_generateMeshesMetaData(JSONWriter& json_writer, XmlNode meshes_node) { // Positionne un numéro de version pour compatibilité avec de futures versions meshes_node.setAttrValue("version", "1"); + + json_writer.writeKey("meshes"); + json_writer.beginArray(); + ISubDomain* sd = m_variable_mng->subDomain(); IParallelMng* pm = m_variable_mng->parallelMng(); IMesh* default_mesh = sd->defaultMesh(); @@ -208,9 +238,10 @@ _generateMeshesMetaData(XmlNode meshes_node) bool do_dump = mesh->properties()->getBool("dump"); // Sauve le maillage s'il est marqué dump ou s'il s'agit du maillage par défaut if (do_dump || mesh == default_mesh) { + JSONWriter::Object o(json_writer); XmlNode mesh_node = XmlElement(meshes_node, "mesh"); - mesh_node.setAttrValue("name", mesh->name()); - mesh_node.setAttrValue("factory-name", mesh->factoryName()); + _writeAttribute(json_writer, mesh_node, "name", mesh->name()); + _writeAttribute(json_writer, mesh_node, "factory-name", mesh->factoryName()); // Indique si le maillage utilise le gestionnaire de parallélisme // séquentiel car dans ce cas en reprise il faut le créer avec le // même gestionnaire. @@ -218,9 +249,11 @@ _generateMeshesMetaData(XmlNode meshes_node) // avec un IParallelMng qui n'est ni séquentiel, ni celui du // sous-domaine. if (is_parallel && mesh->parallelMng() == seq_pm) - mesh_node.setAttrValue("sequential", "true"); + _writeAttribute(json_writer, mesh_node, "sequential", true); } } + + json_writer.endArray(); } /*---------------------------------------------------------------------------*/ @@ -229,13 +262,24 @@ _generateMeshesMetaData(XmlNode meshes_node) String VariableIOWriterMng:: _generateMetaData(const VariableCollection& vars, bool use_hash) { + JSONWriter json_writer(JSONWriter::FormatFlags::None); + ScopedPtrT doc(domutils::createXmlDocument()); XmlNode doc_node = doc->documentNode(); XmlElement root_element(doc_node, "arcane-checkpoint-metadata"); XmlElement variables_node(root_element, "variables"); - _generateVariablesMetaData(variables_node, vars, use_hash); - XmlElement meshes_node(root_element, "meshes"); - _generateMeshesMetaData(meshes_node); + { + JSONWriter::Object o(json_writer); + JSONWriter::Object o2(json_writer, "arcane-checkpoint-metadata"); + json_writer.write("version", static_cast(1)); + _generateVariablesMetaData(json_writer, variables_node, vars, use_hash); + XmlElement meshes_node(root_element, "meshes"); + _generateMeshesMetaData(json_writer, meshes_node); + } + { + // Sauve la sérialisation JSON dans un élément du fichier XML. + XmlElement json_node(root_element, "json", json_writer.getBuffer()); + } String s = doc->save(); info(6) << "META_DATA=" << s; return s; diff --git a/arcane/src/arcane/impl/internal/VariableMng.h b/arcane/src/arcane/impl/internal/VariableMng.h index 30aa19551..dca2a85ad 100644 --- a/arcane/src/arcane/impl/internal/VariableMng.h +++ b/arcane/src/arcane/impl/internal/VariableMng.h @@ -272,8 +272,9 @@ class VariableIOWriterMng void _writeVariables(IDataWriter* writer, const VariableCollection& vars, bool use_hash); String _generateMetaData(const VariableCollection& vars, bool use_hash); - void _generateVariablesMetaData(XmlNode variables_node, const VariableCollection& vars, bool use_hash); - void _generateMeshesMetaData(XmlNode meshes_node); + void _generateVariablesMetaData(JSONWriter& json_writer, XmlNode variables_node, + const VariableCollection& vars, bool use_hash); + void _generateMeshesMetaData(JSONWriter& json_writer, XmlNode meshes_node); static const char* _msgClassName() { return "Variable"; } }; @@ -305,6 +306,7 @@ class VariableIOReaderMng private: VariableMng* m_variable_mng = nullptr; + bool m_is_use_json_metadata = true; private: @@ -312,8 +314,8 @@ class VariableIOReaderMng void _readMetaData(VariableMetaDataList& vmd_list, Span bytes); void _checkHashFunction(const VariableMetaDataList& vmd_list); void _createVariablesFromMetaData(const VariableMetaDataList& vmd_list); - void _readVariablesMetaData(VariableMetaDataList& vmd_list, const XmlNode& variables_node); - void _readMeshesMetaData(const XmlNode& meshes_node); + void _readVariablesMetaData(VariableMetaDataList& vmd_list, JSONValue variables_json, const XmlNode& variables_node); + void _readMeshesMetaData(JSONValue meshes_json, const XmlNode& meshes_node); void _buildFilteredVariableList(VariableReaderMng& vars_read_mng, IVariableFilter* filter); void _finalizeReadVariables(const VariableList& vars_to_read); static const char* _msgClassName() { return "Variable"; } diff --git a/arcane/src/arcane/tests/CMakeLists.txt b/arcane/src/arcane/tests/CMakeLists.txt index 551f31113..8f89f449b 100644 --- a/arcane/src/arcane/tests/CMakeLists.txt +++ b/arcane/src/arcane/tests/CMakeLists.txt @@ -522,10 +522,11 @@ endif() ############### ARCANE_ADD_TEST_PARALLEL(hyoda testHyoda.arc 1) -ARCANE_ADD_TEST(checkpoint_basic testCheckpoint-3.arc -c 3 -m 5) -ARCANE_ADD_TEST(checkpoint_basic2-v1 testCheckpoint-basic2-v1.arc -c 3 -m 5) -ARCANE_ADD_TEST(checkpoint_basic2 testCheckpoint-basic2.arc -c 3 -m 5) -ARCANE_ADD_TEST(checkpoint_basic2-v3 testCheckpoint-basic2-v3.arc -c 3 -m 5) +arcane_add_test(checkpoint_basic testCheckpoint-3.arc -c 3 -m 5) +arcane_add_test(checkpoint_basic2-v1 testCheckpoint-basic2-v1.arc -c 3 -m 5) +arcane_add_test(checkpoint_basic2 testCheckpoint-basic2.arc -c 3 -m 5) +arcane_add_test(checkpoint_basic2-v3 testCheckpoint-basic2-v3.arc -c 3 -m 5) +arcane_add_test(checkpoint_basic2-v3_json_metadata testCheckpoint-basic2-v3.arc -c 3 -m 5 -We,ARCANE_USE_JSON_METADATA,1) arcane_add_test(checkpoint_basic_hash_file testCheckpoint-basic2-v3.arc -c 3 -m 5 -We,ARCANE_HASHDATABASE_DIRECTORY,${CMAKE_CURRENT_BINARY_DIR}/hashdb) if (ARCANE_ENABLE_REDIS_TEST)