Publicaciones
2020 |
Guillaume Rousseau, Roberto Di Cosmo, Stefano Zacchiroli Software provenance tracking at the scale of public source code Artículo de revista Empirical Software Engineering, pp. 1-30, 2020, ISSN: 1573-7616. @article{Rousseau:2020, title = {Software provenance tracking at the scale of public source code}, author = {Guillaume Rousseau and Roberto Di Cosmo and Stefano Zacchiroli }, url = {https://hal.archives-ouvertes.fr/hal-02543794}, doi = {10.1007/s10664-020-09828-5}, issn = {1573-7616}, year = {2020}, date = {2020-05-29}, journal = {Empirical Software Engineering}, pages = {1-30}, abstract = {We study the possibilities to track provenance of software source code artifacts within the largest publicly accessible corpus of publicly available source code, the Software Heritage archive, with over 4 billions unique source code files and 1 billion commits capturing their development histories across 50 million software projects. We perform a systematic and generic estimate of the replication factor across the different layers of this corpus, analysing how much the same artifacts (e.g., SLOC, files or commits) appear in different contexts (e.g., files, commits or source code repositories). We observe a combinatorial explosion in the number of identical source code files across different commits. To discuss the implication of these findings, we benchmark different data models for capturing software provenance information at this scale, and we identify a viable solution, based on the properties of isochrone subgraphs, that is deployable on commodity hardware, is incremental and appears to be maintainable for the foreseeable future. Using these properties, we quantify, at a scale never achieved previously, the growth rate of original, i.e. never-seen-before, source code files and commits, and find it to be exponential over a period of more than 40 years.}, keywords = {}, pubstate = {published}, tppubtype = {article} } We study the possibilities to track provenance of software source code artifacts within the largest publicly accessible corpus of publicly available source code, the Software Heritage archive, with over 4 billions unique source code files and 1 billion commits capturing their development histories across 50 million software projects. We perform a systematic and generic estimate of the replication factor across the different layers of this corpus, analysing how much the same artifacts (e.g., SLOC, files or commits) appear in different contexts (e.g., files, commits or source code repositories). We observe a combinatorial explosion in the number of identical source code files across different commits. To discuss the implication of these findings, we benchmark different data models for capturing software provenance information at this scale, and we identify a viable solution, based on the properties of isochrone subgraphs, that is deployable on commodity hardware, is incremental and appears to be maintainable for the foreseeable future. Using these properties, we quantify, at a scale never achieved previously, the growth rate of original, i.e. never-seen-before, source code files and commits, and find it to be exponential over a period of more than 40 years. |
Roberto Di Cosmo, Marco Danelutto [Rp] Reproducing and replicating the OCamlP3l experiment Artículo de revista ReScience C, 6 (1), 2020. @article{DiCosmo:2020, title = {[Rp] Reproducing and replicating the OCamlP3l experiment}, author = {Roberto Di Cosmo and Marco Danelutto}, url = {https://zenodo.org/record/3763416/files/article.pdf https://rescience.github.io/read/#volume-6-2020}, doi = {10.5281/zenodo.3763416}, year = {2020}, date = {2020-04-30}, journal = {ReScience C}, volume = {6}, number = {1}, abstract = {This article provides a full report on the effort to reproduce the work described in the article “Parallel Functional Programming with Skeletons: the OCamlP3L experiment”, written in 1998. It presented OCamlP3L, a parallel programming system written in the OCaml programming language. It turns out that we found the source code of the OCamlP3L system only in Software Heritage: since it was saved with all its development history, we could perform this reproduction experiment.}, keywords = {}, pubstate = {published}, tppubtype = {article} } This article provides a full report on the effort to reproduce the work described in the article “Parallel Functional Programming with Skeletons: the OCamlP3L experiment”, written in 1998. It presented OCamlP3L, a parallel programming system written in the OCaml programming language. It turns out that we found the source code of the OCamlP3L system only in Software Heritage: since it was saved with all its development history, we could perform this reproduction experiment. |
Paolo Boldi, Antoine Pietri, Sebastiano Vigna, Stefano Zacchiroli Ultra-Large-Scale Repository Analysis via Graph Compression Artículo en actas En preparación SANER 2020: The 27th IEEE International Conference on Software Analysis, Evolution and Reengineering, IEEE, En preparación. @inproceedings{saner-2020-swh-graph, title = {Ultra-Large-Scale Repository Analysis via Graph Compression}, author = {Paolo Boldi and Antoine Pietri and Sebastiano Vigna and Stefano Zacchiroli}, url = {https://www.softwareheritage.org/wp-content/uploads/2020/02/saner-2020-swh-graph.pdf https://upsilon.cc/~zack/research/publications/saner-2020-swh-graph.pdf }, year = {2020}, date = {2020-02-21}, booktitle = {SANER 2020: The 27th IEEE International Conference on Software Analysis, Evolution and Reengineering}, publisher = {IEEE}, abstract = {We consider the problem of mining the development history—as captured by modern version control systems—of ultra-large-scale software archives (e.g., tens of millions software repositories corresponding). We show that graph compression techniques can be applied to the problem, dramatically reducing the hardware resources needed to mine similarly-sized corpus. As a concrete use case we compress the full Software Heritage archive, consisting of 5 billion unique source code files and 1 billion unique commits, harvested from more than 80 million software projects—encompassing a full mirror of GitHub. The resulting compressed graph fits in less than 100 GB of RAM, corresponding to a hardware cost of less than 300 U.S. dollars. We show that the compressed in-memory representation of the full corpus can be accessed with excellent performances, with edge lookup times close to memory random access. As a sample exploitation experiment we show that the compressed graph can be used to conduct clone detection at this scale, benefiting from main memory access speed.}, keywords = {}, pubstate = {forthcoming}, tppubtype = {inproceedings} } We consider the problem of mining the development history—as captured by modern version control systems—of ultra-large-scale software archives (e.g., tens of millions software repositories corresponding). We show that graph compression techniques can be applied to the problem, dramatically reducing the hardware resources needed to mine similarly-sized corpus. As a concrete use case we compress the full Software Heritage archive, consisting of 5 billion unique source code files and 1 billion unique commits, harvested from more than 80 million software projects—encompassing a full mirror of GitHub. The resulting compressed graph fits in less than 100 GB of RAM, corresponding to a hardware cost of less than 300 U.S. dollars. We show that the compressed in-memory representation of the full corpus can be accessed with excellent performances, with edge lookup times close to memory random access. As a sample exploitation experiment we show that the compressed graph can be used to conduct clone detection at this scale, benefiting from main memory access speed. |
Pierre Alliez, Roberto Di Cosmo, Benjamin Guedj, Alain Girault, Mohand-Said Hacid, Arnaud Legrand, Nicolas Rougier Attributing and Referencing (Research) Software: Best Practices and Outlook From Inria Artículo de revista Computing in Science Engineering, 22 (1), pp. 39-52, 2020, ISSN: 1558-366X. @article{2020GtCitation, title = {Attributing and Referencing (Research) Software: Best Practices and Outlook From Inria}, author = {Pierre Alliez and Roberto Di Cosmo and Benjamin Guedj and Alain Girault and Mohand-Said Hacid and Arnaud Legrand and Nicolas Rougier}, url = {https://www.softwareheritage.org/wp-content/uploads/2020/01/2020GtCitation.pdf https://hal.archives-ouvertes.fr/hal-02135891}, doi = {10.1109/MCSE.2019.2949413}, issn = {1558-366X}, year = {2020}, date = {2020-01-01}, journal = {Computing in Science Engineering}, volume = {22}, number = {1}, pages = {39-52}, abstract = {Software is a fundamental pillar of modern scientific research, across all fields and disciplines. However, there is a lack of adequate means to cite and reference software due to the complexity of the problem in terms of authorship, roles, and credits. This complexity is further increased when it is considered over the lifetime of a software that can span up to several decades. Building upon the internal experience of Inria, the French research institute for digital sciences, we provide in this article a contribution to the ongoing efforts in order to develop proper guidelines and recommendations for software citation and reference. Namely, we recommend: first, a richer taxonomy for software contributions with a qualitative scale; second, to put humans at the heart of the evaluation; and third, to distinguish citation from reference.}, keywords = {}, pubstate = {published}, tppubtype = {article} } Software is a fundamental pillar of modern scientific research, across all fields and disciplines. However, there is a lack of adequate means to cite and reference software due to the complexity of the problem in terms of authorship, roles, and credits. This complexity is further increased when it is considered over the lifetime of a software that can span up to several decades. Building upon the internal experience of Inria, the French research institute for digital sciences, we provide in this article a contribution to the ongoing efforts in order to develop proper guidelines and recommendations for software citation and reference. Namely, we recommend: first, a richer taxonomy for software contributions with a qualitative scale; second, to put humans at the heart of the evaluation; and third, to distinguish citation from reference. |
Roberto Di Cosmo, Morane Gruenpeter, Stefano Zacchiroli Referencing Source Code Artifacts: a Separate Concern in Software Citation Artículo de revista Computing in Science & Engineering, 2020, ISSN: 1521-9615. @article{cise-2020-doi, title = {Referencing Source Code Artifacts: a Separate Concern in Software Citation}, author = {Roberto Di Cosmo and Morane Gruenpeter and Stefano Zacchiroli}, url = {https://www.softwareheritage.org/wp-content/uploads/2020/01/2020-CiSE-swhid-1.pdf http://www.dicosmo.org/Articles/2020-CiSE-swhid.pdf https://hal.archives-ouvertes.fr/hal-02446202}, doi = {10.1109/MCSE.2019.2963148}, issn = {1521-9615}, year = {2020}, date = {2020-01-01}, journal = {Computing in Science & Engineering}, publisher = {IEEE}, abstract = {Among the entities involved in software citation, software source code requires special attention, due to the role it plays in ensuring scientific reproducibility. To reference source code we need identifiers that are not only unique and persistent, but also support integrity checking intrinsically. Suitable iden- tifiers must guarantee that denoted objects will always stay the same, without relying on external third parties and administrative processes. We analyze the role of identifiers for digital objects (IDOs), whose properties are different from, and complementary to, those of the various digital identifiers of objects (DIOs) that are today popular building blocks of software and data citation toolchains. We argue that both kinds of identifiers are needed and detail the syntax, semantics, and practical implementation of the persistent identifiers (PIDs) adopted by the Software Heritage project to reference billions of software source code artifacts such as source code files, directories, and commits.}, keywords = {}, pubstate = {published}, tppubtype = {article} } Among the entities involved in software citation, software source code requires special attention, due to the role it plays in ensuring scientific reproducibility. To reference source code we need identifiers that are not only unique and persistent, but also support integrity checking intrinsically. Suitable iden- tifiers must guarantee that denoted objects will always stay the same, without relying on external third parties and administrative processes. We analyze the role of identifiers for digital objects (IDOs), whose properties are different from, and complementary to, those of the various digital identifiers of objects (DIOs) that are today popular building blocks of software and data citation toolchains. We argue that both kinds of identifiers are needed and detail the syntax, semantics, and practical implementation of the persistent identifiers (PIDs) adopted by the Software Heritage project to reference billions of software source code artifacts such as source code files, directories, and commits. |
2019 |
Antoine Pietri, Diomidis Spinellis, Stefano Zacchiroli The Software Heritage Graph Dataset: Public software development under one roof Artículo en actas Proceedings of the 16th International Conference on Mining Software Repositories, pp. 138-142, IEEE Press, 2019. @inproceedings{msr-2019-swh, title = {The Software Heritage Graph Dataset: Public software development under one roof}, author = {Antoine Pietri and Diomidis Spinellis and Stefano Zacchiroli}, url = {https://www.softwareheritage.org/wp-content/uploads/2020/01/msr-2019-swh.pdf https://upsilon.cc/~zack/research/publications/msr-2019-swh.pdf}, doi = {10.1109/MSR.2019.00030}, year = {2019}, date = {2019-05-27}, booktitle = {Proceedings of the 16th International Conference on Mining Software Repositories}, pages = {138-142}, publisher = {IEEE Press}, series = {MSR '19}, abstract = {Software Heritage is the largest existing public archive of software source code and accompanying development history: it currently spans more than five billion unique source code files and one billion unique commits, coming from more than 80 million software projects. This paper introduces the Software Heritage graph dataset: a fully-deduplicated Merkle DAG representation of the Software Heritage archive. The dataset links together file content identifiers, source code directories, Version Control System (VCS) commits tracking evolution over time, up to the full states of VCS repositories as observed by Software Heritage during periodic crawls. The dataset's contents come from major development forges (including GitHub and GitLab), FOSS distributions (e.g., Debian), and language-specific package managers (e.g., PyPI). Crawling information is also included, providing timestamps about when and where all archived source code artifacts have been observed in the wild. The Software Heritage graph dataset is available in multiple formats, including downloadable CSV dumps and Apache Parquet files for local use, as well as a public instance on Amazon Athena interactive query service for ready-to-use powerful analytical processing. Source code file contents are cross-referenced at the graph leaves, and can be retrieved through individual requests using the Software Heritage archive API.}, keywords = {}, pubstate = {published}, tppubtype = {inproceedings} } Software Heritage is the largest existing public archive of software source code and accompanying development history: it currently spans more than five billion unique source code files and one billion unique commits, coming from more than 80 million software projects. This paper introduces the Software Heritage graph dataset: a fully-deduplicated Merkle DAG representation of the Software Heritage archive. The dataset links together file content identifiers, source code directories, Version Control System (VCS) commits tracking evolution over time, up to the full states of VCS repositories as observed by Software Heritage during periodic crawls. The dataset's contents come from major development forges (including GitHub and GitLab), FOSS distributions (e.g., Debian), and language-specific package managers (e.g., PyPI). Crawling information is also included, providing timestamps about when and where all archived source code artifacts have been observed in the wild. The Software Heritage graph dataset is available in multiple formats, including downloadable CSV dumps and Apache Parquet files for local use, as well as a public instance on Amazon Athena interactive query service for ready-to-use powerful analytical processing. Source code file contents are cross-referenced at the graph leaves, and can be retrieved through individual requests using the Software Heritage archive API. |
2018 |
Antoine Pietri, Stefano Zacchiroli Towards Universal Software Evolution Analysis Artículo en actas BENEVOL 2018: The 17th Belgium-Netherlands Software Evolution Workshop, pp. 6-10, 2018, ISSN: 1613-0073. @inproceedings{benevol-2018-swh, title = {Towards Universal Software Evolution Analysis}, author = {Antoine Pietri and Stefano Zacchiroli}, url = {https://www.softwareheritage.org/wp-content/uploads/2020/01/benevol-2018-swh.pdf https://upsilon.cc/~zack/research/publications/benevol-2018-swh.pdf}, issn = {1613-0073}, year = {2018}, date = {2018-12-01}, booktitle = {BENEVOL 2018: The 17th Belgium-Netherlands Software Evolution Workshop}, volume = {2361}, pages = {6-10}, series = {CEUR Workshop Proceedings (CEUR-WS)}, abstract = {Software evolution studies have mostly focused on individual software products, generally developed as Free/Open Source Software (FOSS) projects, and more sparingly on software collections like component and package ecosystems. We argue in this paper that the next step in this organic scale expansion is universal software evolution analysis, i.e., the study of software evolution at the scale of the whole body of publicly available software. We consider the case of Software Heritage, the largest existing archive of publicly available software source code artifacts (more than 5 B unique files archived and 1 B commits, coming from more than 80 M software projects). We propose research requirements that would allow to leverage the Software Heritage archive to study universal software evolution. We discuss the challenges that need to be overcome to address such requirements and outline a research roadmap to do so.}, keywords = {}, pubstate = {published}, tppubtype = {inproceedings} } Software evolution studies have mostly focused on individual software products, generally developed as Free/Open Source Software (FOSS) projects, and more sparingly on software collections like component and package ecosystems. We argue in this paper that the next step in this organic scale expansion is universal software evolution analysis, i.e., the study of software evolution at the scale of the whole body of publicly available software. We consider the case of Software Heritage, the largest existing archive of publicly available software source code artifacts (more than 5 B unique files archived and 1 B commits, coming from more than 80 M software projects). We propose research requirements that would allow to leverage the Software Heritage archive to study universal software evolution. We discuss the challenges that need to be overcome to address such requirements and outline a research roadmap to do so. |
Jean-François Abramatic, Roberto Di Cosmo, Stefano Zacchiroli Building the Universal Archive of Source Code Artículo de revista Communications of the ACM, 61 (10), pp. 29-31, 2018, ISSN: 0001-0782. BibTeX | Enlaces: @article{cacm-2018-software-heritage, title = {Building the Universal Archive of Source Code}, author = {Jean-François Abramatic and Roberto Di Cosmo and Stefano Zacchiroli}, editor = {ACM}, url = {https://cacm.acm.org/magazines/2018/10/231366-building-the-universal-archive-of-source-code/fulltext}, doi = {10.1145/3183558}, issn = {0001-0782}, year = {2018}, date = {2018-10-01}, journal = {Communications of the ACM}, volume = {61}, number = {10}, pages = {29-31}, keywords = {}, pubstate = {published}, tppubtype = {article} } |
Roberto Di Cosmo, Morane Gruenpeter, Stefano Zacchiroli Identifiers for Digital Objects: the Case of Software Source Code Preservation Artículo en actas iPRES 2018 - 15th International Conference on Digital Preservation, 2018. BibTeX | Enlaces: @inproceedings{dicosmo:hal-01865790, title = {Identifiers for Digital Objects: the Case of Software Source Code Preservation}, author = {Roberto Di Cosmo and Morane Gruenpeter and Stefano Zacchiroli}, url = {https://www.softwareheritage.org/wp-content/uploads/2020/01/ipres-2018-swh.pdf https://hal.archives-ouvertes.fr/hal-01865790}, doi = {10.17605/OSF.IO/KDE56}, year = {2018}, date = {2018-09-01}, booktitle = {iPRES 2018 - 15th International Conference on Digital Preservation}, keywords = {}, pubstate = {published}, tppubtype = {inproceedings} } |
Yannick Barborini, Roberto Di Cosmo, Antoine R. Dumont, Morane Gruenpeter, Bruno P. Marmol, Alain Monteil, Jozefina Sadowska, Stefano Zacchiroli The creation of a new type of scientific deposit: Software Miscelánea RDA Eleventh Plenary Meeting, Berlin, Germany, 2018, (poster). BibTeX | Enlaces: @misc{barborini:hal-01738741, title = {The creation of a new type of scientific deposit: Software}, author = {Yannick Barborini and Roberto Di Cosmo and Antoine R. Dumont and Morane Gruenpeter and Bruno P. Marmol and Alain Monteil and Jozefina Sadowska and Stefano Zacchiroli}, url = {https://www.softwareheritage.org/wp-content/uploads/2020/01/barborini-rda-poster.pdf https://hal.inria.fr/hal-01738741}, year = {2018}, date = {2018-03-21}, howpublished = {RDA Eleventh Plenary Meeting, Berlin, Germany}, note = {poster}, keywords = {}, pubstate = {published}, tppubtype = {misc} } |
Yannick Barborini, Roberto Di Cosmo, Antoine R. Dumont, Morane Gruenpeter, Bruno P. Marmol, Alain Monteil, Jozefina Sadowska, Stefano Zacchiroli La création du nouveau type de dépôt scientifique - Le logiciel Miscelánea JSO 2018 - 7es journées Science Ouverte Couperin : 100 % open access : initiatives pour une transition réussie, 2018, (poster). BibTeX | Enlaces: @misc{barborini:hal-01688726, title = {La création du nouveau type de dépôt scientifique - Le logiciel}, author = {Yannick Barborini and Roberto Di Cosmo and Antoine R. Dumont and Morane Gruenpeter and Bruno P. Marmol and Alain Monteil and Jozefina Sadowska and Stefano Zacchiroli}, url = {https://www.softwareheritage.org/wp-content/uploads/2020/01/barborini-jso2018-poster.pdf https://hal.inria.fr/hal-01688726}, year = {2018}, date = {2018-01-22}, howpublished = {JSO 2018 - 7es journées Science Ouverte Couperin : 100 % open access : initiatives pour une transition réussie}, note = {poster}, keywords = {}, pubstate = {published}, tppubtype = {misc} } |
2017 |
Roberto Di Cosmo, Stefano Zacchiroli Software Heritage: Why and How to Preserve Software Source Code Artículo en actas iPRES 2017: 14th International Conference on Digital Preservation, Kyoto, Japan, 2017. BibTeX | Enlaces: @inproceedings{dicosmo:hal-01590958, title = {Software Heritage: Why and How to Preserve Software Source Code}, author = {Roberto Di Cosmo and Stefano Zacchiroli}, url = {https://www.softwareheritage.org/wp-content/uploads/2020/01/ipres-2017-swh.pdf https://hal.archives-ouvertes.fr/hal-01590958}, year = {2017}, date = {2017-09-25}, booktitle = {iPRES 2017: 14th International Conference on Digital Preservation}, address = {Kyoto, Japan}, keywords = {}, pubstate = {published}, tppubtype = {inproceedings} } |