diff options
| author | Paul Bütow <pbuetow@mimecast.com> | 2018-03-01 11:21:26 +0000 |
|---|---|---|
| committer | Paul Bütow <pbuetow@mimecast.com> | 2018-03-01 11:21:26 +0000 |
| commit | 56f8cdff9aaa9bf00c5dc9441a7569374f2cbafb (patch) | |
| tree | b5b440b504b9879e241733fa38d19089fb3377b2 | |
initial commit0.1
79 files changed, 15062 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a122f0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.ko +*.o +docs/html/ +docs/latex/ +ioreplay/ioreplay +systemtap/downloads/ @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..427e7bb --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +DESTDIR=/opt/ioreplay +all: + $(MAKE) -C systemtap + $(MAKE) -C ioreplay +install: + $(MAKE) -C systemtap install + $(MAKE) -C ioreplay install +uninstall: + test ! -z $(DESTDIR) && test -d $(DESTDIR) && rm -Rfv $(DESTDIR) || exit 0 +deinstall: uninstall +clean: + $(MAKE) -C ioreplay clean + $(MAKE) -C systemtap clean +astyle: + $(MAKE) -C ioreplay astyle +loc: + wc -l ./systemtap/src/*.stp ./ioreplay/src/*.{h,c} ./ioreplay/src/*/*.{h,c} | tail -n 1 +doxygen: + doxygen ./docs/doxygen.conf diff --git a/README.md b/README.md new file mode 100644 index 0000000..1bdf632 --- /dev/null +++ b/README.md @@ -0,0 +1,405 @@ +# I/O Replay
+
+## Overview
+
+I/O Replay is an I/O benchmarking tool for Linux based operating systems which captures I/O operations on a (possibly production) server in order to replay the exact same I/O operations on a load test machine.
+
+I/O Replay is operated in 5 steps:
+
+1. Capture: Record all I/O operations over a given period of time to a capture log.
+2. Initialize: Copy the log to a load test machine and initialize the load test environment.
+3. Replay: Drop all OS caches and replay all I/O operations.
+4. Analyze: Look at the OS and hardware stats (throughput, I/O ops, load average) from the run phase and draw conclusions. The aim is to identify possible I/O bottlenecks.
+5. Repeat: Repeat 2-4 times but adjust OS and hardware settings in order to improve I/O performance.
+
+Examples of OS and hardware settings and adjustments:
+
+* Change of system parameters (file system mount options, file system caching, file system type, file system creation flags).
+* Replay the I/O at different speed(s).
+* Replay the I/O with modified pattern(s) (e.g. remove reads from the replay journal).
+* Replay the I/O on different types of hardware.
+
+The file system fragmentation (depending on the file system type and utilisation) might affect I/O performance as well. Therefore, replaying the I/O will not give the exact same result as on a production system. But it provides a pretty good way to determine I/O bottlenecks. As a rule of thumb file system fragmentation will not be an issue, unless the file system begins to fill up. Modern file systems (such as Ext4) will slowly start to suffer from fragmentation and slow down then.
+
+## Benefits
+
+In contrast to traditional I/O benchmarking tools, I/O Replay reproduces real production I/O, and does not rely on a pre-defined set of I/O operations.
+
+Also, I/O Replay only requires a server machine for capturing and another server machine for replaying. A traditional load test environment would usually be a distributed system which can consist of many components and machines. Such a distributed system can become quite complex which makes it difficult to isolate possible I/O bottlenecks. For example in order to trigger I/O events a client application would usually have to call a remote server application. The remote server application itself would query a database and the database would trigger the actual I/O operations in Linux. Furthermore, it is not easy to switch forth and back between hardware and OS settings. For example without a backup and restore procedure a database would most likely be corrupt after reformatting the data partitions with a different file system type.
+
+The benefits of I/O replay are:
+
+* It is easy to determine whether a new hardware type is suitable for an already existing application.
+* It is easy to change OS and hardware for performance tests and optimizations.
+* Findings can be applied to production machines in order to optimize OS configuration and to save hardware costs.
+* Benchmarks are based on production I/O patterns and not on artificial I/O patterns.
+* Log files can be modified to see whether a change in the application behavior would improve I/O performance (without actually touching the application code)
+* Log files could be generated synthetically in order to find out how a new application would perform (even if there isn't any code for the new application yet)
+* It identifies possible flaws in the applications (e.g. Java programs which produce I/O operations on the server machines). Findings can be reported to the corresponding developers so that changes can be introduced to improve the applications I/O performance.
+* It captures I/O in Linux Kernel space (very efficient, no system slowdowns even under heavy I/O load)
+* It replays I/O via a tool developed in C with as little overhead as possible.
+
+# Send in patches
+
+Patches of any kind (bug fixes, new features...) are welcome! I/O Replay is new software and not everything might be perfect yet. Also, I/O Replay is used for a very specific use case at Mimecast. It may need tuning or extension for your use case. It will grow and mature over time.
+
+This is also potentially a great tool just for analysing (not replaying) the I/O, therefore it would be a great opportunity to add more features related to that (e.g. more stats, filters, etc.).
+
+Future work will also include file hole support and I/O support for memory mapped files.
+
+# Getting started
+
+I/O Replay consists of a set of SystemTap kernel modules (capturing I/O) and the tool ``ioreplay`` (replaying I/O). Usually you want to capture I/O from a production machine and want to replay it on a separate load testing machine.
+
+## System requirements
+
+I/O replay has been tested on
+
+* CentOS 7.4 64Bit (latest version, all packages up to date, booted into the installed Kernel)
+* SystemTap (from the default CentOS repository)
+* GCC C-Compiler (from the default CentOS repository)
+
+Before proceeding please ensure that the latest CentOS 7 kernel is installed and running on all machines involved. It should also be ensured that the capture machine and the load test machine have the same mount points mounted. This is to ensure that I/O is being replayed on the corresponding data drives on the load test machine.
+
+## Compiling and installing ioreplay
+
+I/O Replay has to be installed on all machines involved. To install I/O Replay perform the following steps:
+
+```sh
+sudo yum install gcc systemtap yum-utils kernel-devel-$(uname -r)
+sudo debuginfo-install kernel-$(uname -r)
+make && sudo make install
+export PATH=$PATH:/opt/ioreplay/bin
+```
+
+This will install the ``ioreplay`` utility to ``/opt/ioreplay/bin/`` and the SystemTap kernel modules to ``/opt/ioreplay/systemtap/``. Run ``ioreplay -h`` to print out a brief help.
+
+However, best practise is not to install any compilers on a production machine. You can either compile I/O Replay from scratch on all machines involved like shown above or only compile it on a build machine and distribute the ``/opt/ioreplay`` directory to the remaining machines. In the latter case you will also need to install the ``systemtap-runtime`` package as an additional dependency.
+
+In case you decided to deinstall I/O Replay you can do so by running
+
+```sh
+sudo ioreplay -P # purges all test files created by ioreplay
+sudo make uninstall
+```
+
+# Operating I/O Replay
+
+## 1. Capture
+
+The following steps are required to capture all I/O operation of the entire (Linux) system to the file ``io.capture``. For efficiency and security it is only capturing the meta data (amount of bytes written and read) and not the actual data itself. It is also capturing the system time in microseconds and the process IDs (PIDs) and thread IDs (TIDs) used as well as all relevant options and flags of the corresponding I/O syscalls. It will stop capturing automatically after 60 minutes:
+
+ * 1) Stop all applications on the machine. Otherwise the kernel module won't recognize any already opened file handles. Stopping the applications before starting with the capture is essential for tracing the flags in how the files were opened. All I/O operations on unknown file handles will be ignored otherwise.
+ * 2) Run:
+
+```sh
+sudo ioreplay -c ~/io.capture
+```
+
+ * 3) Start all applications again.
+ * 4) To stop capturing I/O type Ctrl+C. Alternatively one hour for the Kernel module to auto exit.
+
+To capture only I/O caused by Java process run:
+
+```sh
+sudo ioreplay -c ~/io.capture -m javaioreplay.ko
+```
+
+To capture the I/O of a specific process run the following respectively:
+
+```sh
+sudo ioreplay -c ~/io.capture -m targetedioreplay.ko -p PID
+```
+
+The resulting capture log looks like this and can be multiple GB in size:
+
+```sh
+t=1511381122062;:,i=7764:8093;:,o=open;:,d=162;:,p=///usr/local/mimecast/someapp/somesubdir/vd11-9:1;:,f=0;:,m=438;:,
+t=1511381122062;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122062;:,i=7764:8093;:,o=read;:,d=162;:,b=12;:,
+t=1511381122062;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122062;:,i=7764:8093;:,o=lseek;:,d=162;:,O=0;:,W=1;:,b=12;:,
+t=1511381122062;:,i=7764:8093;:,o=read;:,d=162;:,b=0;:,
+t=1511381122062;:,i=7764:8093;:,o=close;:,d=162;:,s=0;:,
+t=1511381122062;:,i=7764:8093;:,o=open;:,d=162;:,p=///usr/local/mimecast/someapp/somesubdir/vd11-8:1;:,f=0;:,m=438;:,
+t=1511381122062;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122062;:,i=7764:8093;:,o=read;:,d=162;:,b=12;:,
+t=1511381122062;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122062;:,i=7764:8093;:,o=lseek;:,d=162;:,O=0;:,W=1;:,b=12;:,
+t=1511381122062;:,i=7764:8093;:,o=read;:,d=162;:,b=0;:,
+t=1511381122062;:,i=7764:8093;:,o=close;:,d=162;:,s=0;:,
+t=1511381122062;:,i=7764:8093;:,o=open;:,d=162;:,p=///usr/local/mimecast/someapp/somesubdir/vd11-9:0;:,f=0;:,m=438;:,
+t=1511381122062;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122062;:,i=7764:8093;:,o=read;:,d=162;:,b=12;:,
+t=1511381122062;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122062;:,i=7764:8093;:,o=lseek;:,d=162;:,O=0;:,W=1;:,b=12;:,
+t=1511381122062;:,i=7764:8093;:,o=read;:,d=162;:,b=0;:,
+t=1511381122062;:,i=7764:8093;:,o=close;:,d=162;:,s=0;:,
+t=1511381122063;:,i=7764:8093;:,o=open;:,d=162;:,p=///usr/local/mimecast/someapp/somesubdir/vd11-7:1;:,f=0;:,m=438;:,
+t=1511381122063;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122063;:,i=7764:8093;:,o=read;:,d=162;:,b=12;:,
+t=1511381122063;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122063;:,i=7764:8093;:,o=lseek;:,d=162;:,O=0;:,W=1;:,b=12;:,
+t=1511381122063;:,i=7764:8093;:,o=read;:,d=162;:,b=0;:,
+t=1511381122063;:,i=7764:8093;:,o=close;:,d=162;:,s=0;:,
+t=1511381122063;:,i=7764:8093;:,o=open;:,d=162;:,p=///usr/local/mimecast/someapp/somesubdir/vd11-8:0;:,f=0;:,m=438;:,
+t=1511381122063;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122063;:,i=7764:8093;:,o=read;:,d=162;:,b=12;:,
+t=1511381122063;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122063;:,i=7764:8093;:,o=lseek;:,d=162;:,O=0;:,W=1;:,b=12;:,
+t=1511381122063;:,i=7764:8093;:,o=read;:,d=162;:,b=0;:,
+t=1511381122063;:,i=7764:8093;:,o=close;:,d=162;:,s=0;:,
+t=1511381122063;:,i=7764:8093;:,o=open;:,d=162;:,p=///usr/local/mimecast/someapp/somesubdir/vd11-6:1;:,f=0;:,m=438;:,
+t=1511381122063;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122063;:,i=7764:8093;:,o=read;:,d=162;:,b=12;:,
+t=1511381122063;:,i=7764:8093;:,o=fstat;:,d=162;:,s=0;:,
+t=1511381122063;:,i=7764:8093;:,o=lseek;:,d=162;:,O=0;:,W=1;:,b=12;:,
+```
+
+### Using a RAMdisk (optional)
+
+It is beneficial to write ``io.capture`` to a RAMdisk so that we are not interfering so much with the system I/O:
+
+```sh
+sudo mkdir -p /mnt/ramdisk
+sudo mount -t tmpfs -o size=32g tmpfs /mnt/ramdisk
+```
+
+Make sure that there is enough system memory available for such a RAMdisk and all the processes running on the machine. Eventually, RAM will be taken away from the Linux caches which potentially could decrease system I/O performance. Run the following command to capture to the RAMdisk respectively:
+
+```sh
+sudo ioreplay -c /mnt/ramdisk/io.capture
+```
+
+## 2. Initialize
+
+### 2.1 Pre-process the capture log / generate a replay log
+
+After producing ``io.capture`` it must be pre-processed. The resulting replay log introduces the following changes (for improving replay performance and make the parsing easier):
+
+* Time stamps begin from 0
+* Use of internal opcodes rather than strings (e.g. ``30`` instead of ``open``) for faster parsing.
+* All operations on unknown file handles are _removed_.
+* All incomplete or corrupt lines from the capture file are ignored. There may be corrupt lines in the capture file because SystemTap may skips a very few probe points if it decides that capturing I/O is causing too much overhead.
+* Rewrite of all file paths. ``ioreplay`` adds ``/.ioreplay/NAME`` to all file paths for each file system mount point.
+
+To generate the the replay log ``io.replay`` from the capture log ``io.capture`` run:
+
+```sh
+sudo ioreplay -c io.capture -r io.replay -n NAME -u USER
+```
+
+In which NAME is a freely chosen name and USER must be a valid system user. It is the system user under which the replay test will run. This command also creates all required top level directories such as ``/.ioreplay/NAME/``, ``/mnt/.ioreplay/NAME/`` in all mounted file systems. These are the directories where the replay test will read/write files from/to. These directories will belong to user USER.
+
+``ioreplay`` will filter out many operations, especially all operations on pseudo file systems (e.g. sysfs, procfs), as it does not make a lot of sense to replay I/O on these file systems. Also, I/O operations on unknown file handles will be filtered out as well. This can happen when we start capturing the I/O *after* an application already opened a file. As a result we won't see how the application opened that file. The best practise is to stop all applications on the machine first, start capturing the I/O, and start all applications again. This may be improved in future releases of I/O Replay.
+
+The resulting replay log will look like this: At the first line there is a meta header. It contains information about the test configuration. The meta header is followed by all the I/O operations. At the end of the file is the INIT section. It lists all files (also their sizes) and directories required to be present before replaying the I/O.
+
+
+```sh
+#|num_timelines=509591|num_mapped_pids=19189|num_mapped_fds=4292067|num_lines=55040114|replay_version=1|user=ioreplayuser|name=test0|init_offset=2578735248|
+23|1|1|0|0|30|11|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-9:1|438|0|open@31|
+23|1|1|0|0|0|11|0|fstat@32|
+23|1|1|0|0|10|11|12|read@33|
+23|1|1|0|0|0|11|0|fstat@34|
+23|1|1|0|0|72|11|0|1|12|lseek@35|
+23|1|1|0|0|10|11|0|read@36|
+23|1|1|0|0|50|11|0|close@37|
+23|2|1|0|0|30|12|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-8:1|438|0|open@38|
+23|2|1|0|0|0|12|0|fstat@39|
+23|2|1|0|0|10|12|12|read@40|
+23|2|1|0|0|0|12|0|fstat@41|
+23|2|1|0|0|72|12|0|1|12|lseek@42|
+23|2|1|0|0|10|12|0|read@43|
+23|2|1|0|0|50|12|0|close@44|
+23|3|1|0|0|30|13|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-9:0|438|0|open@45|
+23|3|1|0|0|0|13|0|fstat@46|
+23|3|1|0|0|10|13|12|read@47|
+23|3|1|0|0|0|13|0|fstat@48|
+23|3|1|0|0|72|13|0|1|12|lseek@49|
+23|3|1|0|0|10|13|0|read@50|
+23|3|1|0|0|50|13|0|close@51|
+23|4|1|0|0|30|14|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-7:1|438|0|open@52|
+23|4|1|0|0|0|14|0|fstat@53|
+23|4|1|0|0|10|14|12|read@54|
+23|4|1|0|0|0|14|0|fstat@55|
+23|4|1|0|0|72|14|0|1|12|lseek@56|
+23|4|1|0|0|10|14|0|read@57|
+23|4|1|0|0|50|14|0|close@58|
+23|5|1|0|0|30|15|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-8:0|438|0|open@59|
+23|5|1|0|0|0|15|0|fstat@60|
+23|5|1|0|0|10|15|12|read@61|
+23|5|1|0|0|0|15|0|fstat@62|
+23|5|1|0|0|72|15|0|1|12|lseek@63|
+23|5|1|0|0|10|15|0|read@64|
+23|5|1|0|0|50|15|0|close@65|
+23|6|1|0|0|30|16|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-6:1|438|0|open@66|
+23|6|1|0|0|0|16|0|fstat@67|
+23|6|1|0|0|10|16|12|read@68|
+23|6|1|0|0|0|16|0|fstat@69|
+.
+.
+.
+#INIT
+0|1|688|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/8/b_dv01_11_vd11-11_a|@55290437
+0|1|2592|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/3/b_dv01_11_vd11-11_b|@33907067
+0|1|768|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/d/b_dv01_11_vd11-11_c|@64247527
+0|1|1440|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/0/b_dv01_11_vd11-11_d|@2014896
+0|1|960|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/9/b_dv01_11_vd11-11_e|@17724079
+0|1|928|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/1/b_dv01_11_vd11-11_f|@4534389
+0|1|1712|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/5/b_dv01_11_vd11-11_g|@2738458
+0|1|784|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/b/b_dv01_11_vd11-11_h|@21136612
+0|1|624|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/6/b_dv01_11_vd11-11_i|@24683427
+0|1|672|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/9/b_dv01_11_vd11-11_j|@12584061
+0|1|336|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/5/b_dv01_11_vd11-11_k|@7737434
+0|1|12|/mnt/06/.ioreplay/test0/bmnt/tmp/b|@42498106
+.
+.
+.
+
+```
+
+### 2.2 Initialize the replay test
+
+It is very likely that the replay test wants to access already existing files. Therefore it has to be ensured that all of these exist already before starting the test. To create all files and directories required by the test run the following command:
+
+```sh
+sudo ioreplay -i io.replay
+```
+
+For that ``ioreplay`` makes use of the INIT section in ``io.replay``.
+
+## 3. Replay
+
+It has to be ensured that user USER can open many files and processes. Add the following to ``/etc/security/limits.d/ioreplay.conf``:
+
+```sh
+cat <<END | sudo tee /etc/security/limits.d/ioreplay.conf
+* soft nofile 369216
+* hard nofile 369216
+* soft nproc 30768
+* hard nproc 30768
+END
+```
+
+To replay the log run:
+
+```sh
+sudo ioreplay -r io.replay
+```
+
+It is beneficial to read ``io.replay`` from RAMdisk so that we are not interfering so much with the system I/O.
+
+*Init and replay in one go*
+
+It is posisble to initialise the test and run the test with one single command, just replace option `-r` with `-R`:
+
+```sh
+sudo ioreplay -R io.replay
+```
+
+*Speed factor*
+
+By default `ioreplay` tries to replay all I/O operations as fast as it can. To replay the I/O at a different speed it is possible to configure the speed factor by using the `-s` command line option.
+
+The following pseudo code demonstrates how the speed factor affects the replay speed. Here `current_time` represents the current time while replaying the I/O, `time_in_log` represents the time as logged in `io.replay` and `time_ahead` indicates whether the replay is too quick or not.
+
+```code
+if (speed_factor != 0) {
+ time_ahead = time_in_log / speed_factor - current_time
+ if (time_ahead > 0) {
+ sleep(time_ahead)
+ }
+}
+```
+
+A speed factor of `0` is interpreted as "replay as fast as possible". A speed factor of `1` can be used to replay everything in original speed (same speed as on the original host where the I/O was captured). A speed factor of `2` would double the speed and a speed factor of `0.5` would half the speed.
+In order to replay the I/O in original speed the factor of `1` can be used as follows:
+
+```sh
+sudo ioreplay -R io.replay -s 1
+```
+
+## 4. Analyse
+
+Look at various operating system statistics during the test. Useful commands are for example ``iostat -x 1``, ``dstat --disk`` and ``sudo iotop -o``. Best would be to collect all I/O statistics of all drives to a time series database with graphing capabilities such as Collectd/Graphite/Whisper.
+
+## 5. Repeat
+
+It is important to understand the I/O statistics observed. It is possible to repeat the same test any time again. Each time with different settings applied.
+
+## Cleanup
+
+To purge all temporally data files of all tests run
+
+```sh
+sudo ioreplay -P
+```
+
+Note: It's not required to cleanup any test data manually when you intend to re-run a test or run a new test. During initialization (``-i`` or ``-R`` switch) ``ioreplay`` will automatically move all old data to ``.ioreplay/.trash/`` sub-folders. The data will be ignored there. However, once you intend to completely delete all test files and directories (e.g. you run out of disk space or want to deinstall ``ioreplay`` you should purge them with ``-P`` as shown above.
+
+## Supported file systems
+
+Currently I/O Replay supports replaying I/O on ``ext2``, ``ext3``, ``ext4`` and ``xfs``. However, it should be straightforward add additional file systems.
+
+## Supported syscalls
+
+Currently, these file I/O related syscalls are supported (as of CentOS 7):
+
+```code
+open
+openat
+lseek
+fcntl
+creat
+write
+writev
+unlink
+unlinkat
+rename
+renameat
+renameat2
+read
+readv
+readahead - Initial support only
+readdir
+readlink
+readlinkat
+fdatasync
+fsync
+sync_file_range - Initial support only
+sync
+syncfs
+close
+getdents
+mkdir
+rmdir
+mkdirat
+stat
+statfs - Initial support only
+statfs64 - Initial support only
+fstatfs - Initial support only
+fstatfs64 - Initial support only
+lstat
+fstat
+fstatat
+chmod
+fchmodat
+fchmod
+chown
+chown16
+lchown
+lchown16
+fchown
+fchown16
+fchownat
+mmap2 - Initial support only
+mremap - Initial support only
+munmap - Initial support only
+msync - Initial support only
+exit_group - To detect process termination (closing all open file handles)
+```
+
+## Source code documentation
+
+The documentation of the source code can be generated via the Doxygen Framework. To install doxygen run ``sudo yum install doxygen`` and to generate the documentation run ``make doxygen`` in the top level source directory. Once done, the resulting documentation can be found in the ``docs/html`` subfolder of the project. It is worthwhile to start from ``ioreplay/src/main.c`` and read your way through. Functions are generally documented in the header files. Exceptions are static functions which don't have any separate declarations.
diff --git a/docs/doxygen.conf b/docs/doxygen.conf new file mode 100644 index 0000000..340f731 --- /dev/null +++ b/docs/doxygen.conf @@ -0,0 +1,2432 @@ +# Doxyfile 1.8.13 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all text +# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv +# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv +# for the list of possible encodings. +# The default value is: UTF-8. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "I/O Replay" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = 0.1-BETA + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "An I/O capture and replay benchmarking tool for Linux" + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. + +OUTPUT_DIRECTORY = ./docs + +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. + +CREATE_SUBDIRS = NO + +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. +# The default value is: YES. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. + +ABBREVIATE_BRIEF = "The $name class" \ + "The $name struct" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# doxygen will generate a detailed section even if there is only a brief +# description. +# The default value is: NO. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. +# The default value is: NO. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = YES + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# page for each member. If set to NO, the documentation of a member will be part +# of the file/class/namespace that contains it. +# The default value is: NO. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines. + +ALIASES = + +# This tag can be used to specify a number of word-keyword mappings (TCL only). +# A mapping has the form "name=value". For example adding "class=itcl::class" +# will allow you to use the command class in the itcl::class meaning. + +TCL_SUBST = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = YES + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, Javascript, +# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: +# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: +# Fortran. In the later case the parser tries to guess whether the code is fixed +# or free formatted code, this is the default for Fortran type files), VHDL. For +# instance to make doxygen treat .inc files as Fortran files (default is PHP), +# and .f files as C (default is Fortran), use: inc=Fortran f=C. +# +# Note: For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See http://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 0. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 0 + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. +# The default value is: NO. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. +# The default value is: NO. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. +# The default value is: NO. + +DISTRIBUTE_GROUP_DOC = NO + +# If one adds a struct or class to a group and this option is enabled, then also +# any nested class or struct is added to the same group. By default this option +# is disabled and one has to add nested compounds explicitly via \ingroup. +# The default value is: NO. + +GROUP_NESTED_COMPOUNDS = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. + +TYPEDEF_HIDES_STRUCT = NO + +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = YES + +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be +# included in the documentation. +# The default value is: NO. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO, +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. If set to YES, local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO, only methods in the interface are +# included. +# The default value is: NO. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# (class|struct|union) declarations. If set to NO, these declarations will be +# included in the documentation. +# The default value is: NO. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO, these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. + +INTERNAL_DOCS = YES + +# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file +# names in lower-case letters. If set to YES, upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. +# The default value is: system dependent. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES, the +# scope will be hidden. +# The default value is: NO. + +HIDE_SCOPE_NAMES = NO + +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# append additional text to a page's title, such as Class Reference. If set to +# YES the compound reference will be hidden. +# The default value is: NO. + +HIDE_COMPOUND_REFERENCE= NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. + +SHOW_INCLUDE_FILES = YES + +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. +# The default value is: YES. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. + +SORT_BRIEF_DOCS = YES + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo +# list. This list is created by putting \todo commands in the documentation. +# The default value is: YES. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test +# list. This list is created by putting \test commands in the documentation. +# The default value is: YES. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if <section_label> ... \endif and \cond <section_label> +# ... \endcond blocks. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES, the +# list will mention the files that were used to generate the documentation. +# The default value is: YES. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. + +WARN_IF_UNDOCUMENTED = NO + +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO, doxygen will only warn about wrong or incomplete +# parameter documentation, but not about the absence of documentation. +# The default value is: NO. + +WARN_NO_PARAMDOC = NO + +# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when +# a warning is encountered. +# The default value is: NO. + +WARN_AS_ERROR = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +INPUT = README.md ./ioreplay/src + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: http://www.gnu.org/software/libiconv) for the list of +# possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, +# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. + +FILE_PATTERNS = *.c \ + *.h + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. + +EXAMPLE_PATTERNS = * + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command: +# +# <filter> <input-file> +# +# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = README.md + +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. + +SOURCE_BROWSER = YES + +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. + +INLINE_SOURCES = YES + +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. + +STRIP_CODE_COMMENTS = NO + +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# function all documented functions referencing it will be listed. +# The default value is: NO. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. + +REFERENCES_LINK_SOURCE = YES + +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see http://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the config file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = YES + +# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in +# which the alphabetical index list will be split. +# Minimum value: 1, maximum value: 20, default value: 5. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = + +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefore more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the style sheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# http://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_SECTIONS = YES + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: http://developer.apple.com/tools/xcode/), introduced with +# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a +# Makefile in the HTML output directory. Running make will produce the docset in +# that directory and running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = com.mimecast.IOreplay + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = com.mimecast.Buetow.Paul + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Mimecast + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on +# Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be +# written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_FILE = + +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler (hhc.exe). If non-empty, +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +HHC_LOCATION = + +# The GENERATE_CHI flag controls if a separate .chi index file is generated +# (YES) or that it should be included in the master .chm file (NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +GENERATE_CHI = NO + +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_INDEX_ENCODING = + +# The BINARY_TOC flag controls whether a binary table of contents is generated +# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- +# folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location of Qt's +# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the +# generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_TREEVIEW = YES + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. + +TREEVIEW_WIDTH = 250 + +# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# http://www.mathjax.org) which uses client side Javascript for the rendering +# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = NO + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from http://www.mathjax.org before deployment. +# The default value is: http://cdn.mathjax.org/mathjax/latest. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use <access key> + S +# (what the <access key> is depends on the OS and browser, but it is typically +# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down +# key> to jump into the search results window, the results can be navigated +# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel +# the search. The filter options can be selected when the cursor is inside the +# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys> +# to select a filter and <Enter> or <escape> to activate or cancel the filter +# option. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +SEARCHENGINE = YES + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a web server instead of a web client using Javascript. There +# are two flavors of web server based searching depending on the EXTERNAL_SEARCH +# setting. When disabled, doxygen will generate a PHP script for searching and +# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing +# and searching needs to be provided by external tools. See the section +# "External Indexing and Searching" for details. +# The default value is: NO. +# This tag requires that the tag SEARCHENGINE is set to YES. + +SERVER_BASED_SEARCH = NO + +# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP +# script for searching. Instead the search results are written to an XML file +# which needs to be processed by an external indexer. Doxygen will invoke an +# external search engine pointed to by the SEARCHENGINE_URL option to obtain the +# search results. +# +# Doxygen ships with an example indexer (doxyindexer) and search engine +# (doxysearch.cgi) which are based on the open source search engine library +# Xapian (see: http://xapian.org/). +# +# See the section "External Indexing and Searching" for details. +# The default value is: NO. +# This tag requires that the tag SEARCHENGINE is set to YES. + +EXTERNAL_SEARCH = NO + +# The SEARCHENGINE_URL should point to a search engine hosted by a web server +# which will return the search results when EXTERNAL_SEARCH is enabled. +# +# Doxygen ships with an example indexer (doxyindexer) and search engine +# (doxysearch.cgi) which are based on the open source search engine library +# Xapian (see: http://xapian.org/). See the section "External Indexing and +# Searching" for details. +# This tag requires that the tag SEARCHENGINE is set to YES. + +SEARCHENGINE_URL = + +# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed +# search data is written to a file for indexing by an external tool. With the +# SEARCHDATA_FILE tag the name of this file can be specified. +# The default file is: searchdata.xml. +# This tag requires that the tag SEARCHENGINE is set to YES. + +SEARCHDATA_FILE = searchdata.xml + +# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the +# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is +# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple +# projects and redirect the results back to the right project. +# This tag requires that the tag SEARCHENGINE is set to YES. + +EXTERNAL_SEARCH_ID = + +# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen +# projects other than the one defined by this configuration file, but that are +# all added to the same external search index. Each project needs to have a +# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of +# to a relative location where the documentation can be found. The format is: +# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ... +# This tag requires that the tag SEARCHENGINE is set to YES. + +EXTRA_SEARCH_MAPPINGS = + +#--------------------------------------------------------------------------- +# Configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = YES + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: latex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. +# +# Note that when enabling USE_PDFLATEX this option is only used for generating +# bitmaps for formulas in the HTML output, but not in the Makefile that is +# written to the output directory. +# The default file is: latex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate +# index for LaTeX. +# The default file is: makeindex. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX +# documents. This may be useful for small projects and may help to save some +# trees in general. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used by the +# printer. +# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x +# 14 inches) and executive (7.25 x 10.5 inches). +# The default value is: a4. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +PAPER_TYPE = a4 + +# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names +# that should be included in the LaTeX output. The package can be specified just +# by its name or with the correct syntax as to be used with the LaTeX +# \usepackage command. To get the times font for instance you can specify : +# EXTRA_PACKAGES=times or EXTRA_PACKAGES={times} +# To use the option intlimits with the amsmath package you can specify: +# EXTRA_PACKAGES=[intlimits]{amsmath} +# If left blank no extra packages will be included. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the +# generated LaTeX document. The header should contain everything until the first +# chapter. If it is left blank doxygen will generate a standard header. See +# section "Doxygen usage" for information on how to let doxygen write the +# default header to a separate file. +# +# Note: Only use a user-defined header if you know what you are doing! The +# following commands have a special meaning inside the header: $title, +# $datetime, $date, $doxygenversion, $projectname, $projectnumber, +# $projectbrief, $projectlogo. Doxygen will replace $title with the empty +# string, for the replacement values of the other commands the user is referred +# to HTML_HEADER. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_HEADER = + +# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the +# generated LaTeX document. The footer should contain everything after the last +# chapter. If it is left blank doxygen will generate a standard footer. See +# LATEX_HEADER for more information on how to generate a default footer and what +# special commands can be used inside the footer. +# +# Note: Only use a user-defined footer if you know what you are doing! +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_FOOTER = + +# The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# LaTeX style sheets that are included after the standard style sheets created +# by doxygen. Using this option one can overrule certain style aspects. Doxygen +# will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_EXTRA_STYLESHEET = + +# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the LATEX_OUTPUT output +# directory. Note that the files will be copied as-is; there are no commands or +# markers available. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_EXTRA_FILES = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is +# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will +# contain links (just like the HTML output) instead of page references. This +# makes the output suitable for online browsing using a PDF viewer. +# The default value is: YES. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate +# the PDF file directly from the LaTeX files. Set this option to YES, to get a +# higher quality PDF documentation. +# The default value is: YES. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode +# command to the generated LaTeX files. This will instruct LaTeX to keep running +# if errors occur, instead of asking the user for help. This option is also used +# when generating formulas in HTML. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_BATCHMODE = NO + +# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the +# index chapters (such as File Index, Compound Index, etc.) in the output. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_HIDE_INDICES = NO + +# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source +# code with syntax highlighting in the LaTeX output. +# +# Note that which sources are shown also depends on other settings such as +# SOURCE_BROWSER. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_SOURCE_CODE = NO + +# The LATEX_BIB_STYLE tag can be used to specify the style to use for the +# bibliography, e.g. plainnat, or ieeetr. See +# http://en.wikipedia.org/wiki/BibTeX and \cite for more info. +# The default value is: plain. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_BIB_STYLE = plain + +# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated +# page will contain the date and time when the page was generated. Setting this +# to NO can help when comparing the output of multiple runs. +# The default value is: NO. +# This tag requires that the tag GENERATE_LATEX is set to YES. + +LATEX_TIMESTAMP = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The +# RTF output is optimized for Word 97 and may not look too pretty with other RTF +# readers/editors. +# The default value is: NO. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: rtf. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF +# documents. This may be useful for small projects and may help to save some +# trees in general. +# The default value is: NO. +# This tag requires that the tag GENERATE_RTF is set to YES. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will +# contain hyperlink fields. The RTF file will contain links (just like the HTML +# output) instead of page references. This makes the output suitable for online +# browsing using Word or some other Word compatible readers that support those +# fields. +# +# Note: WordPad (write) and others do not support links. +# The default value is: NO. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's config +# file, i.e. a series of assignments. You only have to provide replacements, +# missing definitions are set to their default value. +# +# See also section "Doxygen usage" for information on how to generate the +# default style sheet that doxygen normally uses. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an RTF document. Syntax is +# similar to doxygen's config file. A template extensions file can be generated +# using doxygen -e rtf extensionFile. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_EXTENSIONS_FILE = + +# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code +# with syntax highlighting in the RTF output. +# +# Note that which sources are shown also depends on other settings such as +# SOURCE_BROWSER. +# The default value is: NO. +# This tag requires that the tag GENERATE_RTF is set to YES. + +RTF_SOURCE_CODE = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for +# classes and files. +# The default value is: NO. + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. A directory man3 will be created inside the directory specified by +# MAN_OUTPUT. +# The default directory is: man. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to the generated +# man pages. In case the manual section does not start with a number, the number +# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is +# optional. +# The default value is: .3. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_EXTENSION = .3 + +# The MAN_SUBDIR tag determines the name of the directory created within +# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by +# MAN_EXTENSION with the initial . removed. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_SUBDIR = + +# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it +# will generate one additional man file for each entity documented in the real +# man page(s). These additional files only source the real man page, but without +# them the man command would be unable to find the correct page. +# The default value is: NO. +# This tag requires that the tag GENERATE_MAN is set to YES. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that +# captures the structure of the code including all documentation. +# The default value is: NO. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: xml. +# This tag requires that the tag GENERATE_XML is set to YES. + +XML_OUTPUT = xml + +# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program +# listings (including syntax highlighting and cross-referencing information) to +# the XML output. Note that enabling this will significantly increase the size +# of the XML output. +# The default value is: YES. +# This tag requires that the tag GENERATE_XML is set to YES. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the DOCBOOK output +#--------------------------------------------------------------------------- + +# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files +# that can be used to generate PDF. +# The default value is: NO. + +GENERATE_DOCBOOK = NO + +# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in +# front of it. +# The default directory is: docbook. +# This tag requires that the tag GENERATE_DOCBOOK is set to YES. + +DOCBOOK_OUTPUT = docbook + +# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the +# program listings (including syntax highlighting and cross-referencing +# information) to the DOCBOOK output. Note that enabling this will significantly +# increase the size of the DOCBOOK output. +# The default value is: NO. +# This tag requires that the tag GENERATE_DOCBOOK is set to YES. + +DOCBOOK_PROGRAMLISTING = NO + +#--------------------------------------------------------------------------- +# Configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an +# AutoGen Definitions (see http://autogen.sf.net) file that captures the +# structure of the code including all documentation. Note that this feature is +# still experimental and incomplete at the moment. +# The default value is: NO. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# Configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module +# file that captures the structure of the code including all documentation. +# +# Note that this feature is still experimental and incomplete at the moment. +# The default value is: NO. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary +# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI +# output from the Perl module output. +# The default value is: NO. +# This tag requires that the tag GENERATE_PERLMOD is set to YES. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely +# formatted so it can be parsed by a human reader. This is useful if you want to +# understand what is going on. On the other hand, if this tag is set to NO, the +# size of the Perl module output will be much smaller and Perl will parse it +# just the same. +# The default value is: YES. +# This tag requires that the tag GENERATE_PERLMOD is set to YES. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file are +# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful +# so different doxyrules.make files included by the same Makefile don't +# overwrite each other's variables. +# This tag requires that the tag GENERATE_PERLMOD is set to YES. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all +# C-preprocessor directives found in the sources and include files. +# The default value is: YES. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES, the include files in the +# INCLUDE_PATH will be searched if a #include is found. +# The default value is: YES. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will be +# used. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that are +# defined before the preprocessor is started (similar to the -D option of e.g. +# gcc). The argument of the tag is a list of macros of the form: name or +# name=definition (no spaces). If the definition and the "=" are omitted, "=1" +# is assumed. To prevent a macro definition from being undefined via #undef or +# recursively expanded use the := operator instead of the = operator. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +PREDEFINED = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this +# tag can be used to specify a list of macro names that should be expanded. The +# macro definition that is found in the sources will be used. Use the PREDEFINED +# tag if you want to use a different macro definition that overrules the +# definition found in the source code. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will +# remove all references to function-like macros that are alone on a line, have +# an all uppercase name, and do not end with a semicolon. Such function macros +# are typically used for boiler-plate code, and will confuse the parser if not +# removed. +# The default value is: YES. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration options related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES tag can be used to specify one or more tag files. For each tag +# file the location of the external documentation should be added. The format of +# a tag file without this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where loc1 and loc2 can be relative or absolute paths or URLs. See the +# section "Linking to external documentation" for more information about the use +# of tag files. +# Note: Each tag file must have a unique name (where the name does NOT include +# the path). If a tag file is not located in the directory in which doxygen is +# run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create a +# tag file that is based on the input files it reads. See section "Linking to +# external documentation" for more information about the usage of tag files. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES, all external class will be listed in +# the class index. If set to NO, only the inherited external classes will be +# listed. +# The default value is: NO. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will be +# listed. +# The default value is: YES. + +EXTERNAL_GROUPS = YES + +# If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in +# the related pages index. If set to NO, only the current project's pages will +# be listed. +# The default value is: YES. + +EXTERNAL_PAGES = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of 'which perl'). +# The default file (with absolute path) is: /usr/bin/perl. + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram +# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to +# NO turns the diagrams off. Note that this option also works with HAVE_DOT +# disabled, but it is recommended to install and use dot, since it yields more +# powerful graphs. +# The default value is: YES. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see: +# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# You can include diagrams made with dia in doxygen documentation. Doxygen will +# then run dia to produce the diagram and insert it in the documentation. The +# DIA_PATH tag allows you to specify the directory where the dia binary resides. +# If left empty dia is assumed to be found in the default search path. + +DIA_PATH = + +# If set to YES the inheritance and collaboration graphs will hide inheritance +# and usage relations if the target is undocumented or is not a class. +# The default value is: YES. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz (see: +# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# Bell Labs. The other options in this section have no effect if this option is +# set to NO +# The default value is: NO. + +HAVE_DOT = NO + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed +# to run in parallel. When set to 0 doxygen will base this on the number of +# processors available in the system. You can set it explicitly to a value +# larger than 0 to get control over the balance between CPU load and processing +# speed. +# Minimum value: 0, maximum value: 32, default value: 0. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_NUM_THREADS = 0 + +# When you want a differently looking font in the dot files that doxygen +# generates you can specify the font name using DOT_FONTNAME. You need to make +# sure dot is able to find the font, which can be done by putting it in a +# standard location or by setting the DOTFONTPATH environment variable or by +# setting DOT_FONTPATH to the directory containing the font. +# The default value is: Helvetica. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_FONTNAME = Helvetica + +# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of +# dot graphs. +# Minimum value: 4, maximum value: 24, default value: 10. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the default font as specified with +# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set +# the path where dot can find it using this tag. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_FONTPATH = + +# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for +# each documented class showing the direct and indirect inheritance relations. +# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a +# graph for each documented class showing the direct and indirect implementation +# dependencies (inheritance, containment, and class references variables) of the +# class with other documented classes. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for +# groups, showing the direct groups dependencies. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +UML_LOOK = NO + +# If the UML_LOOK tag is enabled, the fields and methods are shown inside the +# class node. If there are many fields or methods and many nodes the graph may +# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the +# number of items for each type to make the size more manageable. Set this to 0 +# for no limit. Note that the threshold may be exceeded by 50% before the limit +# is enforced. So when you set the threshold to 10, up to 15 fields may appear, +# but if the number exceeds 15, the total amount of fields shown is limited to +# 10. +# Minimum value: 0, maximum value: 100, default value: 10. +# This tag requires that the tag HAVE_DOT is set to YES. + +UML_LIMIT_NUM_FIELDS = 10 + +# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and +# collaboration graphs will show the relations between templates and their +# instances. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +TEMPLATE_RELATIONS = NO + +# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to +# YES then doxygen will generate a graph for each documented file showing the +# direct and indirect include dependencies of the file with other documented +# files. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +INCLUDE_GRAPH = YES + +# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are +# set to YES then doxygen will generate a graph for each documented file showing +# the direct and indirect include dependencies of the file with other documented +# files. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH tag is set to YES then doxygen will generate a call +# dependency graph for every global function or class method. +# +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. Disabling a call graph can be +# accomplished by means of the command \hidecallgraph. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller +# dependency graph for every global function or class method. +# +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable caller graphs for selected +# functions only using the \callergraph command. Disabling a caller graph can be +# accomplished by means of the command \hidecallergraph. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical +# hierarchy of all classes instead of a textual one. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the +# dependencies a directory has on other directories in a graphical way. The +# dependency relations are determined by the #include relations between the +# files in the directories. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. For an explanation of the image formats see the section +# output formats in the documentation of the dot tool (Graphviz (see: +# http://www.graphviz.org/)). +# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order +# to make the SVG files visible in IE 9+ (other browsers do not have this +# requirement). +# Possible values are: png, jpg, gif, svg, png:gd, png:gd:gd, png:cairo, +# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and +# png:gdiplus:gdiplus. +# The default value is: png. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_IMAGE_FORMAT = png + +# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to +# enable generation of interactive SVG images that allow zooming and panning. +# +# Note that this requires a modern browser other than Internet Explorer. Tested +# and working are Firefox, Chrome, Safari, and Opera. +# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make +# the SVG files visible. Older versions of IE do not have SVG support. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +INTERACTIVE_SVG = NO + +# The DOT_PATH tag can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the \dotfile +# command). +# This tag requires that the tag HAVE_DOT is set to YES. + +DOTFILE_DIRS = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the \mscfile +# command). + +MSCFILE_DIRS = + +# The DIAFILE_DIRS tag can be used to specify one or more directories that +# contain dia files that are included in the documentation (see the \diafile +# command). + +DIAFILE_DIRS = + +# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the +# path where java can find the plantuml.jar file. If left blank, it is assumed +# PlantUML is not used or called during a preprocessing step. Doxygen will +# generate a warning when it encounters a \startuml command in this case and +# will not generate output for the diagram. + +PLANTUML_JAR_PATH = + +# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a +# configuration file for plantuml. + +PLANTUML_CFG_FILE = + +# When using plantuml, the specified paths are searched for files specified by +# the !include statement in a plantuml block. + +PLANTUML_INCLUDE_PATH = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes +# that will be shown in the graph. If the number of nodes in a graph becomes +# larger than this value, doxygen will truncate the graph, which is visualized +# by representing a node as a red box. Note that doxygen if the number of direct +# children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that +# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. +# Minimum value: 0, maximum value: 10000, default value: 50. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs +# generated by dot. A depth value of 3 means that only nodes reachable from the +# root by following a path via at most 3 edges will be shown. Nodes that lay +# further from the root node will be omitted. Note that setting this option to 1 +# or 2 may greatly reduce the computation time needed for large code bases. Also +# note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. +# Minimum value: 0, maximum value: 1000, default value: 0. +# This tag requires that the tag HAVE_DOT is set to YES. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not seem +# to support this out of the box. +# +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) support +# this, this feature is disabled by default. +# The default value is: NO. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page +# explaining the meaning of the various boxes and arrows in the dot generated +# graphs. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot +# files that are used to generate the various graphs. +# The default value is: YES. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_CLEANUP = YES diff --git a/ioreplay/Makefile b/ioreplay/Makefile new file mode 100644 index 0000000..67f4421 --- /dev/null +++ b/ioreplay/Makefile @@ -0,0 +1,36 @@ +#DEBUG=-g3 -ggdb3 -pg +NAME=ioreplay +LIBS=-pthread +CFLAGS=-Wall -std=gnu99 -pedantic +STATIC=#-static +DESTDIR=/opt/ioreplay/bin +SRCS=$(wildcard src/*.c src/*/*.c) +HDRS=$(SRCS:.c=.h) +OBJS=$(SRCS:.c=.o) +all: compile +quick: clean ctags compile sudo_install +cshell: compile + gdb -ex='break main; run' --args ./$(NAME) +test: compile + gdb -ex=run --args ./$(NAME) -U +compile: $(OBJS) + $(CC) $(STATIC) $(DEBUG) $(LIBS) $(OBJS) -o $(NAME) +%.o: %.c %.h + $(CC) $(STATIC) $(DEBUG) $(LIBS) -c $(CFLAGS) $< -o $@ +clean: + rm -v ioreplay ./src/*.o ./src/*/*.o 2>/dev/null || exit 0 +install: + test ! -d $(DESTDIR) && mkdir -p $(DESTDIR) || exit 0 + cp -v $(NAME) $(DESTDIR) + @echo "Don't forget to add $(DESTDIR) to your PATH as follows:" + @echo " export PATH=\$$PATH:$(DESTDIR)" +uninstall: + test ! -z "$(DESTDIR)" && test -f $(DESTDIR)/$(NAME) && rm -v $(DESTDIR)/$(NAME) || exit 0 +deinstall: uninstall +astyle: + astyle -n --style=linux src/*.h src/*/*.h + astyle -n --style=linux src/*.c src/*/*.c +todo: + fgrep ../TODO ./src/* +ctags: + ctags ./src/*.{h,c} ./src/*/*.{h,c} diff --git a/ioreplay/src/capture/capture.c b/ioreplay/src/capture/capture.c new file mode 100644 index 0000000..0ac336b --- /dev/null +++ b/ioreplay/src/capture/capture.c @@ -0,0 +1,99 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "capture.h" + +#include <sys/utsname.h> + +status_e capture_run(options_s *opts) +{ + int status = 0; + struct utsname uts; + + // To make it nicer we should iterate over PATH instead + char *staprun_paths[3] = { + "/usr/bin/staprun", + "/usr/local/bin/staprun", + "/bin/staprun" + }; + int num_staprun_paths = 3; + + if (0 != uname(&uts)) { + Errno("Could not identify release of currently running Kernel!"); + } + + Put("Release of currently running Kernel: %s", uts.release); + char modules_dir[128]; + sprintf(modules_dir, "/opt/ioreplay/systemtap/%s", uts.release); + Put("Changing directory to module path: %s/", modules_dir); + + if (0 != chdir(modules_dir)) { + Errno("Could not change into '%s', please ensure that the compiled " + "SystemTap modules correspond to the currently running Kernel " + "and that these are installed properly!\n", + modules_dir); + } + + if (0 != access(opts->module, R_OK)) { + Errno("Module '%s/%s' can't be read, please make sure that the " + "SystemTap Kernel modules are installed!", + modules_dir, opts->module); + } + + char *staprun_path = NULL; + for (int i = 0; i < num_staprun_paths; ++i) { + if (0 == access(staprun_paths[i], X_OK)) { + staprun_path = staprun_paths[i]; + //Put("SystemTap command path: %s", staprun_path); + break; + } + } + + if (staprun_path == NULL) { + Errno("Can't find 'staprun' command, please ensure to have the SystemTap " + "runtime (usually package 'systemtap-runtime') installed!"); + } + + char staprun_command[128]; + if (opts->pid >= 0) { + sprintf(staprun_command, "%s %s -v -o %s -x %d", staprun_path, opts->module, + opts->capture_file, opts->pid); + } else { + sprintf(staprun_command, "%s %s -v -o %s", staprun_path, opts->module, + opts->capture_file); + } + + Out("NOTICE: It is good practise first to stop all processes, then to "); + Out("start capturing, and then to start all processes again. The reason "); + Out("is that processes may have already open file handles. In that case "); + Out("I/O Replay would be unable to replay these! This may be improved "); + Put("in a future release!"); + Put("To abort capturing now send Ctrl+C, otherwise wait 1h"); + Put("Capturing I/O via: '%s'", staprun_command); + + char buf[1024]; + FILE *fp; + + if ((fp = popen(staprun_command, "r")) == NULL) { + Errno("Unable to invoke staprun command!"); + } + while (fgets(buf, 1024, fp) != NULL) + Out("stapio: %s", buf); + + if (0 != pclose(fp)) { + Error("Problems invoking staprun command!"); + } + + return status; +} diff --git a/ioreplay/src/capture/capture.h b/ioreplay/src/capture/capture.h new file mode 100644 index 0000000..7718d3e --- /dev/null +++ b/ioreplay/src/capture/capture.h @@ -0,0 +1,30 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CAPTURE_H +#define CAPTURE_H + +#include "../defaults.h" +#include "../utils/futils.h" +#include "../options.h" + +/** + * @brief Captures I/O to a .capture file by using stap from SystemTap + * + * @param opts The options object + * @return SUCCESS if everything went fine + */ +status_e capture_run(options_s *opts); + +#endif // CAPTURE_H diff --git a/ioreplay/src/cleanup/cleanup.c b/ioreplay/src/cleanup/cleanup.c new file mode 100644 index 0000000..13c557c --- /dev/null +++ b/ioreplay/src/cleanup/cleanup.c @@ -0,0 +1,30 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cleanup.h" + +#include "../mounts.h" + +status_e cleanup_run(options_s *opts) +{ + drop_root(opts->user); + mounts_s *m = mounts_new(opts); + + if (opts->purge) + mounts_purge(m); + else + mounts_trash(m); + + return SUCCESS; +} diff --git a/ioreplay/src/cleanup/cleanup.h b/ioreplay/src/cleanup/cleanup.h new file mode 100644 index 0000000..127badf --- /dev/null +++ b/ioreplay/src/cleanup/cleanup.h @@ -0,0 +1,29 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CLEANUP_H +#define CLEANUP_H + +#include "../defaults.h" +#include "../options.h" + +/** + * @brief Cleans up all files and directories of a given test + * + * @brief opts The options object + * @return SUCCESS in case everything went fine + */ +status_e cleanup_run(options_s *opts); + +#endif // CLEANUP_H diff --git a/ioreplay/src/datas/amap.c b/ioreplay/src/datas/amap.c new file mode 100644 index 0000000..806a3f8 --- /dev/null +++ b/ioreplay/src/datas/amap.c @@ -0,0 +1,264 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "amap.h" + +/** + * @brief Creates a new array map + * + * @param size The array map size + * @param mmapped true if the memory should be mmapped + * @return The new amap object + */ +static amap_s *_amap_new(long size, bool mmapped) +{ + amap_s *a = NULL; + void ***arrays = NULL; + + // Calculate a multiple of 1024, but at least in size of 'size'. + if (size % 1024 != 0) { + size = 1024*(1+(long)(size/1024)); + } + + if (size < 1) { + Error("Size overflow"); + } + + int num_arrays = size / AMAP_MAX_ARRAY_LENGTH; + + if (mmapped) { + a = Mmapshared(amap_s); + arrays = Cmapshared(num_arrays, void**); + } else { + a = Malloc(amap_s); + arrays = Calloc(num_arrays, void**); + } + + for (int i = 0; i < num_arrays; ++i) { + if (mmapped) { + //Put("%d", AMAP_MAX_ARRAY_LENGTH); + arrays[i] = Cmapshared(AMAP_MAX_ARRAY_LENGTH, void*); + } else { + arrays[i] = Calloc(AMAP_MAX_ARRAY_LENGTH, void*); + } + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + arrays[i][j] = NULL; + } + } + + a->arrays = arrays; + a->num_arrays = num_arrays; + a->size = size; + a->data_destroy = NULL; + a->mmapped = mmapped; + + return a; +} + +/** + * @brief Creates a new array map + * + * @param size The array map size + * @return The new amap object + */ +amap_s* amap_new(const long size) +{ + return _amap_new(size, false); +} + +/** + * @brief Creates a new mmapped array map + * + * @param size The array map size + * @return The new amap object + */ +amap_s* amap_new_mmapped(const long size) +{ + return _amap_new(size, true); +} + +/** + * @brief Destroys a mmap object + * + * @a The new amap object + */ +void amap_destroy(amap_s* a) +{ + if (!a) { + return; + } + + // Don't bother, the mmapped version of amap will stay alive until + // process terminations. And after process termination everything + // will be cleaned up automatically by Linux. + if (a->mmapped) { + return; + } + + for (int i = 0; i < a->num_arrays; ++i) { + if (a->data_destroy) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) + if (a->arrays[i][j]) { + a->data_destroy(a->arrays[i][j]); + } + } + free(a->arrays[i]); + } + free(a->arrays); + free(a); +} + +/** + * @brief Resets a mmap object + * + * This resets all entries to NULL. + * + * @a The new amap object + */ +void amap_reset(amap_s* a) +{ + for (int i = 0; i < a->num_arrays; ++i) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + if (a->data_destroy) { + if (a->arrays[i][j]) { + a->data_destroy(a->arrays[i][j]); + } + } + a->arrays[i][j] = NULL; + } + } +} + +int amap_set(amap_s *a, const long position, void* value) +{ + if (position >= a->size) + return -1; + int which_array = position / AMAP_MAX_ARRAY_LENGTH; + int array_pos = position % AMAP_MAX_ARRAY_LENGTH; + a->arrays[which_array][array_pos] = value; + return 0; +} + +void* amap_get(amap_s *a, const long position) +{ + if (position >= a->size) + return NULL; + int which_array = position / AMAP_MAX_ARRAY_LENGTH; + int array_pos = position % AMAP_MAX_ARRAY_LENGTH; + return a->arrays[which_array][array_pos]; +} + +void* amap_unset(amap_s *a, const long position) +{ + if (position >= a->size) + return NULL; + int which_array = position / AMAP_MAX_ARRAY_LENGTH; + int array_pos = position % AMAP_MAX_ARRAY_LENGTH; + void *value = a->arrays[which_array][array_pos]; + a->arrays[which_array][array_pos] = NULL; + return value; +} + +void amap_run_cb(amap_s *a, void (*cb)(void *data)) +{ + for (int i = 0; i < a->num_arrays; ++i) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + if (a->arrays[i][j]) + cb(a->arrays[i][j]); + } + } +} + +void amap_print(amap_s* a) +{ + Put("amap_s (%p):", (void*)a); + Put("\tmmapped: %d", a->mmapped); + Put("\tmax_array_length: %d", AMAP_MAX_ARRAY_LENGTH); + Put("\tnum_arrays: %d", a->num_arrays); + Put("\tsize: %lu", a->size); + Out("\toccupied slots: "); + for (int i = 0; i < a->num_arrays; ++i) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + if (a->arrays[i][j] != NULL) { + Out("%d:%d ", i, j); + } + } + } + Out("\n"); +} + +void _amap_test(amap_s *a) +{ + assert(0 == amap_set(a, 0, (void*)10)); + assert(0 == amap_set(a, 1, (void*)11)); + assert(0 == amap_set(a, 2, (void*)12)); + assert(0 == amap_set(a, 3, (void*)a)); + assert(10 == (long) amap_get(a, 0)); + assert(11 == (long) amap_get(a, 1)); + assert(12 == (long) amap_get(a, 2)); + assert(a == amap_get(a, 3)); + + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH-1, (void*) 23)); + assert(23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH-1)); + + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH, (void*) 42)); + assert(42 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH)); + + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2-1, (void*) (23+42))); + assert(42+23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH*2-1)); + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2, (void*) 23)); + + + assert(NULL == amap_get(a, 1024*1024*9-1)); + assert(0 == amap_set(a, 1024*1024*9-1, (void*) 0x1)); + assert(0x1 == (long) amap_get(a, 1024*1024*9-1)); + assert(0x1 == (long) amap_unset(a, 1024*1024*9-1)); + assert(NULL == amap_get(a, 1024*1024*9-1)); + + assert(0 == amap_set(a, 1024*1024*9, (void*) 100)); + assert(100 == (long) amap_get(a, 1024*1024*9)); + + assert(0 == amap_set(a, 1024*1024*9+1, (void*) 101)); + assert(101 == (long) amap_get(a, 1024*1024*9+1)); + + assert(0 == amap_set(a, 1024*1024*10-2, (void*) 102)); + assert(102 == (long) amap_get(a, 1024*1024*10-2)); + + assert(0 == amap_set(a, 1024*1024*10-1, a)); + assert(a == amap_get(a, 1024*1024*10-1)); + //amap_print(a); + + assert(a == amap_unset(a, 1024*1024*10-1)); + assert(a != amap_unset(a, 1024*1024*10-1)); + //amap_print(a); +} + +void amap_test(void) +{ + // First test the non-mmapped version + amap_s* a = amap_new(1024*1024*10); + _amap_test(a); + amap_destroy(a); + + // Now test the mapped version + a = amap_new_mmapped(1024*1024*10); + _amap_test(a); + amap_destroy(a); + + // Another test with non-alligned size + a = amap_new(1024*1024*10+1); + _amap_test(a); + amap_destroy(a); +} + diff --git a/ioreplay/src/datas/amap.h b/ioreplay/src/datas/amap.h new file mode 100644 index 0000000..882a7c5 --- /dev/null +++ b/ioreplay/src/datas/amap.h @@ -0,0 +1,49 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef AMAP_H +#define AMAP_H + +#include "../defaults.h" + +#define AMAP_MAX_ARRAY_LENGTH 1024*8 + +/** + * @brief Implements an array map data structure + * + * This array map can hold a HUGE amount of entries by allocating multiple + * smaller arrays. There are two version of the amap data structure available: + * a memory mapped (mmap) and a normal version. The memory mapped version can + * be used for IPC between various processes. + */ +typedef struct amap_s_ { + void*** arrays; /**< The pointers to the amap arrays */ + int num_arrays; /**< The amount of arrays used in the amap */ + long size; /**< The total size/capacity of the amap */ + bool mmapped; /**< True if amap is memory mapped */ + void (*data_destroy)(void *data); /**< Callback to destroy all elements */ +} amap_s; + +amap_s* amap_new(const long size); +amap_s* amap_new_mmapped(const long size); +int amap_set(amap_s *a, const long position, void* value); +void* amap_get(amap_s *a, const long position); +void* amap_unset(amap_s *a, const long position); +void amap_print(amap_s *a); +void amap_destroy(amap_s *a); +void amap_reset(amap_s *a); +void amap_run_cb(amap_s *a, void (*cb)(void *data)); +void amap_test(void); + +#endif // AMAP_H diff --git a/ioreplay/src/datas/btree.c b/ioreplay/src/datas/btree.c new file mode 100644 index 0000000..da5da48 --- /dev/null +++ b/ioreplay/src/datas/btree.c @@ -0,0 +1,169 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "btree.h" + +btree_s* btree_new() +{ + btree_s *b = Malloc(btree_s); + *b = (btree_s) { + .root = NULL, .size = 0 + }; + return b; +} + +void btree_destroy(btree_s* b) +{ + if (b->root) + btreelem_destroy_r(b->root); + free(b); +} + +void btree_destroy2(btree_s* b) +{ + if (b->root) + btreelem_destroy_r2(b->root); + free(b); +} + +int btree_insert(btree_s* b, int key, void *data) +{ + int ret = 1; + + if (b->root == NULL) { + b->root = btreelem_new(key, data); + ret = 0; + } else { + ret = btreelem_insert_r(b->root, key, data); + } + + if (ret == 0) { + b->size++; + } + + return ret; +} + +void* btree_get(btree_s* b, int key) +{ + if (b->root == NULL) + return NULL; + + return btreelem_get_r(b->root, key); +} + +void btree_print(btree_s* b) +{ + btreelem_print_r(b->root, 0); +} + +btreelem_s* btreelem_new(int key, void *data) +{ + btreelem_s *e = Malloc(btreelem_s); + *e = (btreelem_s) { + .key = key, .data = data, .left = NULL, .right = NULL + }; + return e; +} + +void btreelem_destroy_r(btreelem_s* e) +{ + if (e->left) { + btreelem_destroy_r(e->left); + } + if (e->right) { + btreelem_destroy_r(e->right); + } + + free(e); +} + +void btreelem_destroy_r2(btreelem_s* e) +{ + if (e->left) + btreelem_destroy_r(e->left); + if (e->right) + btreelem_destroy_r(e->right); + if (e->data) + btree_destroy(e->data); + + free(e); +} + +int btreelem_insert_r(btreelem_s* e, int key, void *data) +{ + int ret = 0; + + if (e->key == key) { + ret = 1; + } + + else if (e->key > key) { + if (e->left == NULL) { + e->left = btreelem_new(key, data); + } else { + ret = btreelem_insert_r(e->left, key, data); + } + } + + else { + if (e->right == NULL) { + e->right = btreelem_new(key, data); + } else { + ret = btreelem_insert_r(e->right, key, data); + } + } + + return ret; +} + +void* btreelem_get_r(btreelem_s* e, int key) +{ + void *data = NULL; + + if (e->key == key) { + data = e->data; + } + + else if (e->key > key) { + if (e->left) { + data = btreelem_get_r(e->left, key); + } + } + + else { + if (e->right) { + data = btreelem_get_r(e->right, key); + } + } + + return data; +} + +void btreelem_print_r(btreelem_s* e, int depth) +{ + for (int i = 0; i < depth; ++i) { + Out(" "); + } + Put("%d\n", e->key); + + if (e->left) { + btreelem_print_r(e->left, depth); + } + + if (e->right) { + btreelem_print_r(e->right, depth+1); + } +} + diff --git a/ioreplay/src/datas/btree.h b/ioreplay/src/datas/btree.h new file mode 100644 index 0000000..55da560 --- /dev/null +++ b/ioreplay/src/datas/btree.h @@ -0,0 +1,52 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BTREE_H +#define BTREE_H + +#include "../defaults.h" + +/** + * @brief This defines an element of the binary tree data structure + */ +typedef struct btreelem_ { + struct btreelem_ *left; /**< The next element to the left */ + struct btreelem_ *right; /**< The next element to the right */ + int key; /**< The key of the element */ + void *data; /**< A pointer to the data stored in this element */ +} btreelem_s; + +/** + * @brief This defines a binary tree data structure. + */ +typedef struct btree_s_ { + btreelem_s *root; /**< The root element */ + int size; /**< The current size of the binary tree */ +} btree_s; + +btree_s* btree_new(); +void btree_destroy(btree_s *b); +void btree_destroy2(btree_s *b); +int btree_insert(btree_s *b, int key, void *data); +void* btree_get(btree_s *b, int key); +void btree_print(btree_s *b); + +btreelem_s* btreelem_new(int key, void *data); +void btreelem_destroy_r(btreelem_s *e); +void btreelem_destroy_r2(btreelem_s *e); +int btreelem_insert_r(btreelem_s *e, int key, void *data); +void* btreelem_get_r(btreelem_s *e, int key); +void btreelem_print_r(btreelem_s *e, int depth); + +#endif // BTREE_H diff --git a/ioreplay/src/datas/hmap.c b/ioreplay/src/datas/hmap.c new file mode 100644 index 0000000..d0e1d70 --- /dev/null +++ b/ioreplay/src/datas/hmap.c @@ -0,0 +1,364 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hmap.h" + +#define _Using_string_keys h->keys != NULL + +unsigned int hmap_get_addr(hmap_s *h, char *key) +{ + unsigned long hash = 5381; + int len = strlen(key); + + for (int i = 0; i < len; ++i) { + hash = ((hash << 5) + hash) + key[i]; /* hash * 33 + c */ + } + + return (unsigned int) (hash % h->size); +} + +unsigned int hmap_get_addr_l(hmap_s *h, const long key) +{ + return (unsigned int) (key % h->size); +} + +hmap_s *_hmap_new(unsigned int init_size) +{ + hmap_s *h = Malloc(hmap_s); + + h->size = init_size; + h->data = Calloc(init_size, void*); + h->l = Calloc(init_size, list_s*); + h->data_destroy = NULL; + h->keys = NULL; + h->keys_l = NULL; + + Mset(h->data, 0, init_size, void*); + Mset(h->l, 0, init_size, list_s*); + + return h; +} + +hmap_s *hmap_new(unsigned int init_size) +{ + hmap_s *h = _hmap_new(init_size); + h->keys = Calloc(init_size, char*); + Mset(h->keys, 0, init_size, char*); + + return h; +} + +hmap_s *hmap_new_l(unsigned int init_size) +{ + hmap_s *h = _hmap_new(init_size); + h->keys_l = Calloc(init_size, int); + Mset(h->keys_l, -1, init_size, int); + + return h; +} + +void hmap_destroy(hmap_s *h) +{ + for (int i = 0; i < h->size; ++i) { + if (h->l[i]) { + list_s *l = h->l[i]; + if (h->data_destroy) + l->data_destroy = h->data_destroy; + list_destroy(h->l[i]); + } + if (h->data[i] && h->data_destroy) { + h->data_destroy(h->data[i]); + } + } + + free(h->data); + if (h->keys) + free(h->keys); + if (h->keys_l) + free(h->keys_l); + free(h->l); + free(h); + + return; +} + +int hmap_insert(hmap_s *h, char *key, void *data) +{ + if (data == NULL) { + Error("insert data can not be NULL"); + } + + int addr = hmap_get_addr(h, key); + + if (h->data[addr]) { + + if (strcmp(key, h->keys[addr]) == 0) { + // Key already exists + return 0; + } + + // There is already data, collision, create a linked list + list_s *l = h->l[addr] = list_new(); + list_key_insert(l, h->keys[addr], h->data[addr]); + list_key_insert(l, key, data); + + // Not needed anymore, as the elements are in the linked list now. + free(h->keys[addr]); + h->data[addr] = h->keys[addr] = NULL; + + return 1; + + } else if (h->l[addr]) { + // There was a collision at this address before. Insert + // the element to the linked list. Returns 0 if key is already + // in the list (no additional insert made) or 1 otherwise. + return list_key_insert(h->l[addr], key, data); + } + + // New entry on a collision free address + h->data[addr] = data; + h->keys[addr] = Clone(key); + + return 1; +} + +int hmap_insert_l(hmap_s *h, const long key, void *data) +{ + if (data == NULL) { + Error("insert data can not be NULL"); + } + + int addr = hmap_get_addr_l(h, key); + + if (h->data[addr]) { + + if (key == h->keys_l[addr]) { + // Key already exists + return 0; + } + + // There is already data, collision, create a linked list + list_s *l = h->l[addr] = list_new_l(); + list_key_insert_l(l, h->keys_l[addr], h->data[addr]); + list_key_insert_l(l, key, data); + + // Not needed anymore, as the elements are in the linked list now. + h->data[addr] = NULL; + h->keys_l[addr] = -1; + + return 1; + + } else if (h->l[addr]) { + // There was a collision at this address before. Insert + // the element to the linked list. Returns 0 if key is already + // in the list (no additional insert made) or 1 otherwise. + return list_key_insert_l(h->l[addr], key, data); + } + + // New entry on a collision free address + h->data[addr] = data; + h->keys_l[addr] = key; + + return 1; +} + +void* hmap_remove(hmap_s *h, char *key) +{ + int addr = hmap_get_addr(h, key); + + if (h->data[addr] != NULL) { + void *data = h->data[addr]; + free(h->keys[addr]); + h->data[addr] = h->keys[addr] = NULL; + return data; + + } else if (h->l[addr] != NULL) { + // There was a collision at this address before. Remove + // the element to the linked list. Returns the object if key is + // already in the list (no additional insert made) or NULL + // otherwise. + return list_key_remove(h->l[addr], key); + } + + // Key is not present + return NULL; +} + +void* hmap_remove_l(hmap_s *h, const long key) +{ + int addr = hmap_get_addr_l(h, key); + + if (h->data[addr] != NULL) { + void *data = h->data[addr]; + h->data[addr] = NULL; + h->keys_l[addr] = -1; + return data; + + } else if (h->l[addr] != NULL) { + // There was a collision at this address before. Remove + // the element to the linked list. Returns the object if key is + // already in the list (no additional insert made) or NULL + // otherwise. + return list_key_remove_l(h->l[addr], key); + } + + // Key is not present + return NULL; +} + +void* hmap_get(hmap_s *h, char *key) +{ + int addr = hmap_get_addr(h, key); + if (h->data[addr] && strcmp(h->keys[addr], key) == 0) { + return h->data[addr]; + + } else if (h->l[addr]) { + return list_key_get(h->l[addr], key); + } + + return NULL; +} + +void* hmap_get_l(hmap_s *h, const long key) +{ + int addr = hmap_get_addr_l(h, key); + if (h->data[addr] && h->keys_l[addr] == key) { + return h->data[addr]; + + } else if (h->l[addr]) { + return list_key_get_l(h->l[addr], key); + } + + return NULL; +} + +void hmap_run_cb(hmap_s* h, void (*cb)(void *data)) +{ + for (int i = 0; i < h->size; ++i) { + if (h->l[i]) { + list_s *l = h->l[i]; + list_run_cb(l, cb); + } + if (h->data[i]) { + cb(h->data[i]); + } + } +} + +void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_) +{ + for (int i = 0; i < h->size; ++i) { + if (h->l[i]) { + list_s *l = h->l[i]; + list_run_cb2(l, cb, data_); + } + if (h->data[i]) { + cb(h->data[i], data_); + } + } +} + +void hmap_print(hmap_s *h) +{ + for (int i = 0; i < h->size; ++i) { + if (h->data[i]) { + if (_Using_string_keys) { + Put("hmap:%p addr:%d key:'%s'", (void*)h, i, h->keys[i]); + } else { + Put("hmap:%p addr:%d key:%d", (void*)h, i, h->keys_l[i]); + } + } else if (h->l[i]) { + Put("hmap:%p addr:%d LIST", (void*)h, i); + list_print(h->l[i]); + } + } +} + +static void _hmap_test(hmap_s *h) +{ + void* somedata = (void*)h; + + assert(1 == hmap_insert(h, "someval", (void*)23)); + assert(1 == hmap_insert(h, "another value", (void*)123)); + + assert(1 == hmap_insert(h, "mimecast", somedata)); + assert(0 == hmap_insert(h, "mimecast", somedata)); + assert(1 == hmap_insert(h, "is", somedata)); + assert(1 == hmap_insert(h, "hiring", somedata)); + + assert(NULL != hmap_get(h, "mimecast")); + assert(NULL == hmap_get(h, "Mimecast")); + + assert(NULL != hmap_remove(h, "mimecast")); + assert(NULL == hmap_remove(h, "mimecast")); + + assert(1 == hmap_insert(h, "mimecast", somedata)); + assert(NULL != hmap_get(h, "mimecast")); + + assert(23 == (long)hmap_get(h, "someval")); + assert(23 == (long)hmap_get(h, "someval")); + + assert(123 == (long)hmap_remove(h, "another value")); + assert(0 == (long)hmap_remove(h, "another value")); + assert(NULL == hmap_get(h, "another value")); + + //hmap_print(h); +} + +static void _hmap_test_l(hmap_s *h) +{ + void* somedata = (void*)h; + + assert(1 == hmap_insert_l(h, 1, (void*)23)); + + assert(1 == hmap_insert_l(h, 1, (void*)23)); + assert(1 == hmap_insert_l(h, 5, (void*)123)); + + assert(1 == hmap_insert_l(h, 3, somedata)); + assert(0 == hmap_insert_l(h, 3, somedata)); + assert(1 == hmap_insert_l(h, 4, somedata)); + assert(1 == hmap_insert_l(h, 6, somedata)); + + assert(NULL != hmap_get_l(h, 3)); + assert(NULL == hmap_get_l(h, 7)); + + assert(NULL != hmap_remove_l(h, 3)); + assert(NULL == hmap_remove_l(h, 3)); + + assert(1 == hmap_insert_l(h, 3, somedata)); + assert(NULL != hmap_get_l(h, 3)); + + assert(23 == (long)hmap_get_l(h, 1)); + assert(23 == (long)hmap_get_l(h, 1)); + + assert(123 == (long)hmap_remove_l(h, 5)); + assert(0 == (long)hmap_remove_l(h, 5)); + assert(NULL == hmap_get_l(h, 5)); +} + +void hmap_test(void) +{ + hmap_s* h = hmap_new(1024); + _hmap_test(h); + hmap_destroy(h); + + h = hmap_new(2); + _hmap_test(h); + hmap_destroy(h); + + h = hmap_new_l(1024); + _hmap_test_l(h); + hmap_print(h); + hmap_destroy(h); +} diff --git a/ioreplay/src/datas/hmap.h b/ioreplay/src/datas/hmap.h new file mode 100644 index 0000000..9d1978b --- /dev/null +++ b/ioreplay/src/datas/hmap.h @@ -0,0 +1,56 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HMAP_H +#define HMAP_H + +#include "../defaults.h" +#include "list.h" + +/** + * @brief A hash map data structure + * + * There are two version of this hmap data structure. One version is utilising + * string keys and the other one is utilising long keys. + * + * On hash collision the data structure will make use of a "named" linked list, + * whereas every member of the linked list has either a string key or a long + * key associated. + */ +typedef struct hmap_s_ { + char **keys; /**< List of all keys, NULL if nothing at a address */ + int *keys_l; /**< Same as keys, but for long keys */ + void **data; /**< Pointers to the stored data, NULL if nothing there */ + list_s **l; /**< Pointers to the linked lists, used on hash collision */ + void (*data_destroy)(void *data); /**< Callback to destroy all data */ + unsigned int size; /**< Size of the hmap */ +} hmap_s; + +hmap_s* hmap_new(unsigned int init_size); +hmap_s* hmap_new_l(unsigned int init_size); +void hmap_destroy(hmap_s* h); +void hmap_run_cb(hmap_s* h, void (*cb)(void *data)); +void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_); +int hmap_insert_l(hmap_s* h, const long key, void *data); +int hmap_insert(hmap_s* h, char* key, void *data); +void* hmap_remove_l(hmap_s* h, const long key); +void* hmap_remove(hmap_s* h, char* key); +void* hmap_get_l(hmap_s* h, const long key); +void* hmap_get(hmap_s* h, char* key); +unsigned int hmap_get_addr_l(hmap_s* h, const long key); +unsigned int hmap_get_addr(hmap_s* h, char* key); +void hmap_print(hmap_s* h); +void hmap_test(void); + +#endif // HMAP_H diff --git a/ioreplay/src/datas/list.c b/ioreplay/src/datas/list.c new file mode 100644 index 0000000..9cc78db --- /dev/null +++ b/ioreplay/src/datas/list.c @@ -0,0 +1,279 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "list.h" + + +list_s *list_new() +{ + list_s *l = Malloc(list_s); + *l = (list_s) { + .first = NULL, .data_destroy = NULL + }; + return l; +} + +list_s *list_new_l() +{ + return list_new(); +} + +void list_destroy(list_s *l) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key) + free(current->key); + if (current->data && l->data_destroy) + l->data_destroy(current->data); + list_elem_s *next = current->next; + free(current); + current = next; + } + + free(l); +} + +int list_key_insert(list_s *l, char *key, void *data) +{ + list_elem_s *current = l->first; + + while (current) { + // Already in the list + if (strcmp(current->key, key) == 0) + return 0; + current = current->next; + } + + list_elem_s *e = Malloc(list_elem_s); + + e->prev = NULL; + e->next = l->first; + e->key = Clone(key); + e->key_l = -1; + e->data = data; + + if (l->first) { + l->first->prev = e; + l->first = e; + + } else { + l->first = e; + } + + return 1; +} + +int list_key_insert_l(list_s *l, const long key, void *data) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key_l == key) + return 0; + current = current->next; + } + + list_elem_s *e = Malloc(list_elem_s); + + e->prev = NULL; + e->next = l->first; + e->key = NULL; + e->key_l = key; + e->data = data; + + if (l->first) { + l->first->prev = e; + l->first = e; + + } else { + l->first = e; + } + + return 1; +} + +void _list_elem_remove(list_s *l, list_elem_s *e) +{ + if (l->first == e) { + list_elem_s *first = e->next; + if (first) + first->prev = NULL; + l->first = first; + + } else { + list_elem_s *prev = e->prev; + list_elem_s *next = e->next; + + prev->next = next; + if (next) + next->prev = prev; + } + + if (e->key) + free(e->key); + free(e); +} + +void* list_key_remove(list_s *l, char *key) +{ + list_elem_s *current = l->first; + + while (current) { + if (strcmp(current->key, key) == 0) { + void *data = current->data; + _list_elem_remove(l, current); + return data; + } + current = current->next; + } + + return NULL; +} + +void* list_key_remove_l(list_s *l, const long key) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key_l == key) { + void *data = current->data; + _list_elem_remove(l, current); + return data; + } + current = current->next; + } + + return NULL; +} + +void* list_key_get(list_s *l, char *key) +{ + list_elem_s *current = l->first; + + while (current) { + if (strcmp(current->key, key) == 0) + return current->data; + current = current->next; + } + + return NULL; +} + +void* list_key_get_l(list_s *l, const long key) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key_l == key) + return current->data; + current = current->next; + } + + return NULL; +} + +void list_run_cb(list_s* l, void (*cb)(void *data)) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->data) + cb(current->data); + current = current->next; + } +} + +void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->data) + cb(current->data, data_); + current = current->next; + } +} + +void list_print(list_s *l) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key != NULL) { + Put("list:%p key:'%s' data:%p", (void*)l, + current->key, current->data); + } else { + Put("list:%p key:%ld data:%p", (void*)l, + current->key_l, current->data); + } + current = current->next; + } +} + +void list_test(void) +{ + list_s *l = list_new(); + void* somedata = (void*)l; + + assert(1 == list_key_insert(l, "foo", (void*)1)); + assert(1 == list_key_insert(l, "bar", (void*)2)); + assert(1 == list_key_insert(l, "baz", (void*)3)); + assert(2 == (long)list_key_remove(l, "bar")); + assert(1 == (long)list_key_remove(l, "foo")); + assert(3 == (long)list_key_remove(l, "baz")); + + assert(1 == list_key_insert(l, "I/O replay", somedata)); + assert(1 == list_key_insert(l, "for", somedata)); + assert(1 == list_key_insert(l, "benchmarking your server", somedata)); + assert(0 == list_key_insert(l, "for", somedata)); + + assert(NULL != list_key_get(l, "benchmarking your server")); + assert(NULL == list_key_get(l, "Mimecast")); + + assert(NULL != list_key_remove(l, "benchmarking your server")); + assert(NULL == list_key_remove(l, "benchmarking your server")); + assert(1 == list_key_insert(l, "benchmarking your server", somedata)); + + assert(1 == list_key_insert(l, "MiMecast", (void*)42)); + assert(42 == (long)list_key_get(l, "MiMecast")); + + l = list_new_l(); + + assert(1 == list_key_insert_l(l, 1, (void*)1)); + assert(1 == list_key_insert_l(l, 2, (void*)2)); + assert(1 == list_key_insert_l(l, 3, (void*)3)); + assert(1 == (long)list_key_get_l(l, 1)); + assert(1 == (long)list_key_remove_l(l, 1)); + assert(1 != (long)list_key_remove_l(l, 1)); + assert(3 == (long)list_key_remove_l(l, 3)); + + assert(1 == list_key_insert_l(l, 1234, somedata)); + assert(1 == list_key_insert_l(l, 13, somedata)); + assert(1 == list_key_insert_l(l, 666, somedata)); + assert(0 == list_key_insert_l(l, 13, somedata)); + + assert(NULL != list_key_get_l(l, 666)); + assert(NULL == list_key_get_l(l, 777)); + + assert(NULL != list_key_remove_l(l, 666)); + assert(NULL == list_key_remove_l(l, 666)); + assert(1 == list_key_insert_l(l, 666, somedata)); + + assert(1 == list_key_insert_l(l, 42, (void*)42)); + assert(42 == (long)list_key_get_l(l, 42)); + + //list_print(l); +} diff --git a/ioreplay/src/datas/list.h b/ioreplay/src/datas/list.h new file mode 100644 index 0000000..385333c --- /dev/null +++ b/ioreplay/src/datas/list.h @@ -0,0 +1,56 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef LIST_H +#define LIST_H + +#include "../defaults.h" + +/** + * @brief Definition of a linked list element + */ +typedef struct list_elem_s_ { + struct list_elem_s_ *prev; /**< The previous element */ + struct list_elem_s_ *next; /**< The next element */ + char *key; /**< The key of the lemenet */ + long key_l; /**< The same as key, but for long keys */ + void *data; /**< Pointer to the stored data */ +} list_elem_s; + +/** + * @brief Definition of a named linked list data structure + * + * There are two version of this list data structure. One version is utilising + * string keys and the other one is utilising long keys. + */ +typedef struct list_s_ { + list_elem_s *first; /**< The first element, NULL if list empty */ + void (*data_destroy)(void *data); /**< Callback to destroy all data */ +} list_s; + +list_s* list_new(); +list_s* list_new_l(); +void list_destroy(list_s* l); +void list_run_cb(list_s* l, void (*cb)(void *data)); +void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_); +int list_key_insert(list_s* l, char *key, void *data); +int list_key_insert_l(list_s* l, const long key, void *data); +void* list_key_remove(list_s* l, char *key); +void* list_key_remove_l(list_s* l, const long key); +void* list_key_get(list_s* l, char *key); +void* list_key_get_l(list_s* l, const long key); +void list_print(list_s* l); +void list_test(); + +#endif // LIST_H diff --git a/ioreplay/src/datas/rbuffer.c b/ioreplay/src/datas/rbuffer.c new file mode 100644 index 0000000..c019e6c --- /dev/null +++ b/ioreplay/src/datas/rbuffer.c @@ -0,0 +1,147 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rbuffer.h" + +rbuffer_s *rbuffer_new(const int size) +{ + rbuffer_s *r = Malloc(rbuffer_s); + + r->size = size; + r->read_pos = size-1; + r->write_pos = 0; + r->ring = Calloc(size, void*); + + Mset(r->ring, 0, size, void*); + + return r; +} + +void rbuffer_destroy(rbuffer_s *r) +{ + if (r) { + free(r->ring); + free(r); + } +} + +bool rbuffer_insert(rbuffer_s* r, void *data) +{ + if (r->write_pos == r->read_pos) + // Ring buffer is full + return false; + + r->ring[r->write_pos] = data; + r->write_pos = (r->write_pos+1) % r->size; + + return true; +} + +bool rbuffer_has_next(rbuffer_s* r) +{ + sig_atomic_t read_pos = (r->read_pos+1) % r->size; + + if (read_pos == r->write_pos) + // No more items to read, buffer is empty + { + return false; + } + + return true; +} + +void* rbuffer_get_next(rbuffer_s* r) +{ + sig_atomic_t read_pos = (r->read_pos+1) % r->size; + + if (read_pos == r->write_pos) + // No more items to read, buffer is empty + { + return NULL; + } + + void *data = r->ring[read_pos]; + r->ring[read_pos] = NULL; + r->read_pos = read_pos; + + return data; +} + +void rbuffer_print(rbuffer_s* r) +{ + Put("rbuffer_s (%p):", (void*)r); + Put("\tsize: %d", (int)r->size); + Put("\tread_pos: %d", r->read_pos); + Put("\twrite_pos: %d", r->write_pos); + Out("\toccupied slots: "); + for (int i = 0; i < r->size; ++i) + if (r->ring[i]) { + Out("%d:%p ", i, r->ring[i]); + } + Out("\n"); +} + +void rbuffer_test(void) +{ + rbuffer_s *r = rbuffer_new(5); + assert(NULL == rbuffer_get_next(r)); + + assert(rbuffer_insert(r, (void*)1)); + assert(rbuffer_insert(r, (void*)2)); + assert(rbuffer_insert(r, (void*)3)); + assert(rbuffer_insert(r, (void*)4)); + assert(!rbuffer_insert(r, (void*)5)); + rbuffer_print(r); + + assert(rbuffer_has_next(r)); + assert(1 == (long) rbuffer_get_next(r)); + assert(2 == (long) rbuffer_get_next(r)); + assert(3 == (long) rbuffer_get_next(r)); + assert(4 == (long) rbuffer_get_next(r)); + assert(!rbuffer_has_next(r)); + assert(NULL == rbuffer_get_next(r)); + + assert(rbuffer_insert(r, (void*)1)); + assert(1 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)2)); + assert(2 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)3)); + assert(3 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)4)); + assert(4 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)5)); + assert(5 == (long) rbuffer_get_next(r)); + assert(NULL == rbuffer_get_next(r)); + rbuffer_print(r); + + assert(rbuffer_insert(r, (void*)1)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)2)); + assert(1 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)3)); + assert(2 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)4)); + assert(3 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)5)); + rbuffer_print(r); + assert(4 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(5 == (long) rbuffer_get_next(r)); + assert(NULL == rbuffer_get_next(r)); + + rbuffer_destroy(r); +} diff --git a/ioreplay/src/datas/rbuffer.h b/ioreplay/src/datas/rbuffer.h new file mode 100644 index 0000000..fa634de --- /dev/null +++ b/ioreplay/src/datas/rbuffer.h @@ -0,0 +1,102 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RBUFFER_H +#define RBUFFER_H + +#include "signal.h" + +#include "../defaults.h" + +/** + * @brief An atomic ring buffer data type definition + * + * This data structure can be used for the common producer/consumer problem. + * As long as there is only max one producer thread and max one consumer thread + * it can be used without any mutex locking. All the operations are atomic. + */ +typedef struct rbuffer_s_ { + /** + * The positions are atomic, means the ring buffer can be accessed from + * multiple threads concurrently (one producer and one consumer thread). + * This is the current read position. + */ + sig_atomic_t read_pos; + /** + * This is the current write position. + */ + sig_atomic_t write_pos; + /** + * Holds the pointers to the actual ring data stored in the ring buffer + */ + void **ring; + /** + * Determines how many elements the ring buffer can hold. The capacity + * will be size-1 though, as we need one empty slot. + */ + int size; +} rbuffer_s; + +/** + * @brief Creates a new ring buffer + * + * @param size The size of the ring buffer + * @return The new ring buffer object + */ +rbuffer_s* rbuffer_new(const int size); + +/** + * @brief Destroys a ring buffer + * + * @param r The ring buffer object + */ +void rbuffer_destroy(rbuffer_s* r); + +/** + * @brief Inserts data pointer to the ring buffer + * + * @param r The ring buffer object + * @param data The data pointer + */ +bool rbuffer_insert(rbuffer_s* r, void *data); + +/** + * @brief Determines whether there is any data in the ring buffer + * + * @param r The ring buffer object + * @return True if there is any data, false otherwise + */ +bool rbuffer_has_next(rbuffer_s* r); + +/** + * @brief Returns and removes the next element from the ring buffer + * + * @param r The ring buffer object + * @return The data pointer + */ +void* rbuffer_get_next(rbuffer_s* r); + +/** + * @brief Prints a ring buffer + * + * @param r The ring buffer object + */ +void rbuffer_print(rbuffer_s* r); + +/** + * @brief Unit tests the ring buffer + */ +void rbuffer_test(void); + +#endif // RBUFFER_H diff --git a/ioreplay/src/datas/stack.c b/ioreplay/src/datas/stack.c new file mode 100644 index 0000000..94e83e3 --- /dev/null +++ b/ioreplay/src/datas/stack.c @@ -0,0 +1,85 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "stack.h" + + +stack_s *stack_new() +{ + stack_s *s = Malloc(stack_s); + *s = (stack_s) { + .top = NULL, .size = 0 + }; + return s; +} + +void stack_destroy(stack_s *s) +{ + stack_elem_s *current = s->top; + + while (current) { + stack_elem_s *next = current->next; + free(current); + current = next; + } + + free(s); +} + +void stack_push(stack_s *s, void *data) +{ + stack_elem_s *new_top = Malloc(stack_elem_s); + + *new_top = (stack_elem_s) { + .next = s->top, + .data = data + }; + + s->top = new_top; + s->size++; +} + +void* stack_pop(stack_s *s) +{ + if (s->top == NULL) { + return NULL; + } + + stack_elem_s *old_top = s->top; + + void *data = old_top->data; + s->top = old_top->next; + free(old_top); + s->size--; + + return data; +} + +int stack_is_empty(stack_s *s) +{ + return s->top == NULL; +} + +stack_s* stack_new_reverse_from(stack_s *s) +{ + stack_s* r = stack_new(); + + while (!stack_is_empty(s)) { + stack_push(r, stack_pop(s)); + } + + stack_destroy(s); + + return r; +} diff --git a/ioreplay/src/datas/stack.h b/ioreplay/src/datas/stack.h new file mode 100644 index 0000000..87e0974 --- /dev/null +++ b/ioreplay/src/datas/stack.h @@ -0,0 +1,43 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef STACK_H +#define STACK_H + +#include "../defaults.h" + +/** + * @brief Definition of a stack element + */ +typedef struct stack_elem_s_ { + struct stack_elem_s_ *next; /**< The next element */ + void *data; /**< Pointer to the stored data in the current element */ +} stack_elem_s; + +/** + * @brief Definition of a stack data structure + */ +typedef struct stack_s_ { + stack_elem_s *top; /**< The top element of the stack, NULL if empty */ + unsigned long size; /**< A count how many elements are in the stack */ +} stack_s; + +stack_s* stack_new(); +stack_s* stack_new_reverse_from(stack_s* s); +void stack_destroy(stack_s* s); +void stack_push(stack_s* s, void *data); +void* stack_pop(stack_s* s); +int stack_is_empty(stack_s* s); + +#endif // STACK_H diff --git a/ioreplay/src/defaults.h b/ioreplay/src/defaults.h new file mode 100644 index 0000000..c607b95 --- /dev/null +++ b/ioreplay/src/defaults.h @@ -0,0 +1,50 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DEFAULTS_H +#define DEFAULTS_H + +#include "utils/utils.h" + +/** Version of the supported .capture format */ +#define CAPTURE_VERSION 1 +/** Version of the supported .replay format */ +#define REPLAY_VERSION 1 +/** Max amount of tokens per line in the .capture file */ +#define MAX_TOKENS 10 +/** Max line length in either .capture or .replay file */ +#define MAX_LINE_LEN 1024*8 +/** Controls how many tasks can be queued and buffered per worker thread */ +#define TASK_BUFFER_PER_THREAD 512 +/** Version of I/O Replay */ +#define IOREPLAY_VERSION "0.1" +/** Copyright information */ +#define IOREPLAY_COPYRIGHT "Mimecast 2017, 2018 (c)" + +// The following are for debugging purposes only + +//#define NO_IOOP +//#define THREAD_DEBUG +//#define LOG_FILTERED + +/** + * @brief Return status codes + */ +typedef enum status_e_ { + SUCCESS, /**< Great success! */ + UNKNOWN, /**< Unknown return status :-/ */ + ERROR, /**< An error happened :-( */ +} status_e; + +#endif // DEFAULTS_H diff --git a/ioreplay/src/generate/generate.c b/ioreplay/src/generate/generate.c new file mode 100644 index 0000000..ff1be94 --- /dev/null +++ b/ioreplay/src/generate/generate.c @@ -0,0 +1,235 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generate.h" + +#include "../meta/meta.h" +#include "gtask.h" +#include "gwriter.h" +#include "gparser.h" + +#include <fcntl.h> + +#define _MAX_PROCESSES 1024*1024*10 + +#define _Perc_filtered (g->num_lines_filtered / (g->lineno/100.0)) + +generate_s* generate_new(options_s *opts) +{ + generate_s *g = Malloc(generate_s); + + g->writer = NULL; + g->lineno = 0; + g->name = opts->name; + g->replay_fd = NULL; + g->mps = mounts_new(opts); + g->num_lines_filtered = 0; + g->num_vsizes = 0; + g->start_time = -1; + g->pid_map = amap_new(_MAX_PROCESSES); + g->vsize_map = hmap_new(_MAX_PROCESSES); + g->mmap_map = hmap_new(1024*1024); + g->vfd_buffer = rbuffer_new(1024); + g->num_mapped_pids = 0; + g->num_mapped_fds = 10; + g->opts = opts; + g->reuse_queue = rbuffer_new(1024); + g->replay_fd = Fopen(opts->replay_file, "w"); + + return g; +} + +void generate_destroy(generate_s *g) +{ + // TODO: Also clean the contets of these maps + amap_destroy(g->pid_map); + hmap_destroy(g->vsize_map); + hmap_destroy(g->mmap_map); + rbuffer_destroy(g->vfd_buffer); + mounts_destroy(g->mps); + + gtask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(g->reuse_queue))) + gtask_destroy(task); + rbuffer_destroy(g->reuse_queue); + + fclose(g->replay_fd); + free(g); +} + + +status_e generate_run(options_s *opts) +{ + generate_s *g = generate_new(opts); + Put("Parsing file %s, writing output to %s", opts->capture_file, + opts->replay_file); + FILE *capture_fd = Fopen(opts->capture_file, "r"); + + size_t len = 0; + ssize_t read; + char *line = NULL; + + drop_root(opts->user); + + // Reserve first few bytes for meta information + meta_s *meta = meta_new(g->replay_fd); + meta_reserve(meta); + + // The writer will write the .replay file + gwriter_s *writer = gwriter_new(g); + + // The parser will parse every line of the .capture file + gparser_s *parser = gparser_new(g); + + g->writer = writer; + + // Start one writer and one parser thread! + gparser_start(parser); + gwriter_start(writer); + + Out("Processing, it may take a while: "); + + // Process each line of the .capture file. Determine line by line whether + // the I/O operation makes sense or not. It might be that SystemTap skipped + // some I/O ops due to system overload or other issues. The result is that + // some lines may be corrupt or contain I/O operations on unknown file + // handles. It could also be that there are operations on unknown + // file handles such as sockets etc. These will be all filtered out by + // either the parser or the writer thread! + + while ((read = getline(&line, &len, capture_fd)) != -1) { + if (0 > ++g->lineno) { + Error("lineno:%lu Line number overflow", g->lineno); + } + if (strlen(line) >= MAX_LINE_LEN) { + Error("lineno:%lu Exceeded max line length", g->lineno); + } + + // Create a new generate task (try to reuse a task object)... + gtask_s *t = rbuffer_get_next(g->reuse_queue); + if (!t) { + t = gtask_new(g); + } else if (t->ret != 0) { + g->num_lines_filtered++; + } + gtask_init(t, line, g->lineno); + + // ...pass it to the parser queue + while (!rbuffer_insert(parser->queue, t)) + usleep(100); + + if (g->lineno % 1000000 == 0) { + Out(" %lu (filtered:%.2lf%%)", g->lineno, _Perc_filtered); + } + } + + Put("\nDone reading input file!"); + + Put("Waiting for parser thread..."); + gparser_terminate(parser); + gparser_destroy(parser); + + Put("Waiting for writer thread..."); + gwriter_terminate(writer); + gwriter_destroy(writer); + + // Retrieve all left over processed tasks to collect the + // statistics! + gtask_s *t; + while (NULL != (t = rbuffer_get_next(g->reuse_queue))) { + if (t->ret != 0) + g->num_lines_filtered++; + gtask_destroy(t); + } + + Put("Processed %lu lines in total, had to filter out %.2lf%%", + g->lineno, _Perc_filtered); + + Put("Writing init section to '%s'...", opts->replay_file); + fprintf(g->replay_fd, "#INIT\n"); + off_t init_offset = ftello(g->replay_fd); + hmap_run_cb(g->vsize_map, generate_write_init_cb); + + Put("Writing meta header to '%s'...", opts->replay_file); + meta_write_start(meta); + + // The meta header is being written to the first line of the .replay + // file and used by ioreplay to do various things (e.g. initializing + // the test correctly, creating the internal data structures with the + // correct sizes etc. + + meta_write_l(meta, "replay_version", REPLAY_VERSION); + meta_write_l(meta, "init_offset", init_offset); + + meta_write_s(meta, "user", opts->user); + meta_write_s(meta, "name", opts->name); + + meta_write_l(meta, "num_vsizes", g->num_vsizes); + meta_write_l(meta, "num_mapped_pids", g->num_mapped_pids); + meta_write_l(meta, "num_mapped_fds", g->num_mapped_fds); + meta_write_l(meta, "num_lines", g->lineno - g->num_lines_filtered); + + meta_destroy(meta); + fclose(capture_fd); + + Put("Generating '%s' done", opts->replay_file); + generate_destroy(g); + + return SUCCESS; +} + +void generate_write_init_cb(void *data) +{ + vsize_s *l = data; + generate_s *g = l->generate; + + if (l->required && strlen(l->path) > 0) { + fprintf(g->replay_fd, "%d|%d|%ld|%s|\n", + l->is_dir, l->is_file, -l->vsize_deficit, l->path); + } +} + +vsize_s* generate_vsize_by_path(generate_s *g, gtask_s *t, + char *path) +{ + vsize_s *v = NULL; + + if (!path && t) + path = t->path; + + Error_if(!path, "No path specified"); + v = hmap_get(g->vsize_map, path); + + if (!v) { + v = vsize_new(path, ++g->num_vsizes, g); + hmap_insert(g->vsize_map, path, v); + } + + if (t) + t->vsize = v; + + return v; +} + +void generate_gprocess_by_realpid(generate_s *g, gtask_s *t) +{ + // Get the virtual process data object from the virtual PID space. + t->gprocess = amap_get(g->pid_map, t->pid); + if (t->gprocess == NULL) { + t->gprocess = gprocess_new(t->pid, ++g->num_mapped_pids); + if (amap_set(g->pid_map, t->pid, t->gprocess)) { + Error("lineno:%lu Can not insert PID %ld", t->lineno, t->pid); + } + } +} diff --git a/ioreplay/src/generate/generate.h b/ioreplay/src/generate/generate.h new file mode 100644 index 0000000..cf096d2 --- /dev/null +++ b/ioreplay/src/generate/generate.h @@ -0,0 +1,112 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GENERATE_H +#define GENERATE_H + +#include "gwriter.h" +#include "../datas/amap.h" +#include "../datas/hmap.h" +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "../mounts.h" +#include "../options.h" + +// Forward declarations (header include hell) +struct gtask_s_; + +/** + * @brief The generate object definition + * + * This is the general data structure required to generate a .replay file from + * the .capture file. + */ +typedef struct generate_s_ { + long lineno; /**< The current line number */ + long num_lines_filtered; /**< The amount of lines filtered out */ + long start_time; /**< The start time from the .capture file */ + char *name; /**< The name of the test specified by the user */ + FILE *replay_fd; /**< The fd of the .replay file */ + mounts_s *mps; /**< The mounts object */ + hmap_s *mmap_map; /**< mmap address mappings */ + amap_s *pid_map; /**< A map of all virtual process objects */ + unsigned long num_mapped_pids; /**< The amount of mapped PIDs */ + unsigned long num_mapped_fds; /**< The amount of mapped FDs */ + hmap_s *vsize_map; /**< A hash map of all virtual size objects */ + unsigned long num_vsizes; /**< The amount of virtual sizes */ + options_s *opts; /**< A pointer to the options object */ + rbuffer_s *vfd_buffer; /**< A virtual fd buffer, for reusing these */ + rbuffer_s *reuse_queue; /**< A task buffer, for reusing these */ + struct gwriter_s_ *writer; /**< A pointer to the writer object */ +} generate_s; + +/** + * @brief Creates a new generate object + * + * @param opts The options object + * @return The new generate object + */ +generate_s* generate_new(options_s *opts); + +/** + * @brief Destroys a generate object + * + * @param g The generate object to destroy + */ +void generate_destroy(generate_s* g); + +/** + * @brief Generates a .replay file from a .capture file + * + * @param opts The options object + * @return SUCCESS on success + */ +status_e generate_run(options_s *opts); + +/** + * @brief Callback to write the INIT section to the .replay file + * + * This function writes a list of all pre-required + * paths to the .replay file. That then can be used + * by ioreplay to initialise the test enironment. + * + * @param data A pointer to the vsize timestamp object + */ +void generate_write_init_cb(void *data); + +/** + * @brief Retrieves the virtual size object of a given path + * + * A new one will be created in case there is no such virtual size object yet. + * + * @param g The generate object + * @param t The task object (vfd will be stored to t->vfd) + * @param path The file path + * @return The virtual size object + */ +vsize_s* generate_vsize_by_path(generate_s *g, struct gtask_s_ *t, + char *path); + +/** + * @brief Retrieves the virtual process object of a given real PID + * + * A new one will be created in case there is no such virtual process object + * yet. + * + * @param g The generate object + * @param t The task object (vfd will be stored to t->gprocess) + */ +void generate_gprocess_by_realpid(generate_s *g, struct gtask_s_ *t); + +#endif // GENERATE_H diff --git a/ioreplay/src/generate/gioop.c b/ioreplay/src/generate/gioop.c new file mode 100644 index 0000000..01701bc --- /dev/null +++ b/ioreplay/src/generate/gioop.c @@ -0,0 +1,838 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gioop.h" + +status_e gioop_run(gwriter_s *w, gtask_s *t) +{ + status_e ret = SUCCESS; + + // There was already an error in the parser (parser.c) processing this + // task! Don't process it futher. + if (t->ret != SUCCESS) { + Cleanup(t->ret); + } + + generate_s *g = w->generate; + + // Get the virtual process data object from the virtual PID space and store + // a pointer to it to t->gprocess + generate_gprocess_by_realpid(g, t); + + // One of the open syscalls may openes a file handle succesfully + if (Eq(t->op, "open")) { + Cleanup(gioop_open(w, t, g)); + + } else if (Eq(t->op, "openat")) { + Cleanup(gioop_openat(w, t, g)); + + } else if (Eq(t->op, "creat")) { + Cleanup(gioop_creat(w, t, g)); + } + + // Get the virtual file descriptor of a given real fd and store a pointer + // to it to t->vfd. + if (t->has_fd) { + ret = gprocess_vfd_by_realfd(t->gprocess, t); + Cleanup_unless(SUCCESS, ret); + } + + + if (Eq(t->op, "close")) { + Cleanup(gioop_close(w, t, g)); + + } else if (Eq(t->op, "stat")) { + Cleanup(gioop_stat(w, t, g)); + + } else if (Eq(t->op, "statfs")) { + Cleanup(gioop_statfs(w, t, g)); + + } else if (Eq(t->op, "statfs64")) { + Cleanup(gioop_statfs64(w, t, g)); + + } else if (Eq(t->op, "fstat")) { + Cleanup(gioop_fstat(w, t, g)); + + } else if (Eq(t->op, "fstatat")) { + Cleanup(gioop_fstatat(w, t, g)); + + } else if (Eq(t->op, "fstatfs")) { + Cleanup(gioop_fstatfs(w, t, g)); + + } else if (Eq(t->op, "fstatfs64")) { + Cleanup(gioop_fstatfs64(w, t, g)); + + } else if (Eq(t->op, "rename")) { + Cleanup(gioop_rename(w, t, g)); + + } else if (Eq(t->op, "renameat")) { + Cleanup(gioop_renameat(w, t, g)); + + } else if (Eq(t->op, "renameat2")) { + Cleanup(gioop_renameat2(w, t, g)); + + } else if (Eq(t->op, "read")) { + Cleanup(gioop_read(w, t, g)); + + } else if (Eq(t->op, "readv")) { + Cleanup(gioop_readv(w, t, g)); + + } else if (Eq(t->op, "readahead")) { + Cleanup(gioop_readahead(w, t, g)); + + } else if (Eq(t->op, "readdir")) { + Cleanup(gioop_readdir(w, t, g)); + + } else if (Eq(t->op, "readlink")) { + Cleanup(gioop_readlink(w, t, g)); + + } else if (Eq(t->op, "readlinkat")) { + Cleanup(gioop_readlinkat(w, t, g)); + + } else if (Eq(t->op, "write")) { + Cleanup(gioop_write(w, t, g)); + + } else if (Eq(t->op, "writev")) { + Cleanup(gioop_writev(w, t, g)); + + } else if (Eq(t->op, "lseek")) { + Cleanup(gioop_lseek(w, t, g)); + + } else if (Eq(t->op, "getdents")) { + Cleanup(gioop_getdents(w, t, g)); + + } else if (Eq(t->op, "mkdir")) { + Cleanup(gioop_mkdir(w, t, g)); + + } else if (Eq(t->op, "rmdir")) { + Cleanup(gioop_rmdir(w, t, g)); + + } else if (Eq(t->op, "mkdirat")) { + Cleanup(gioop_mkdirat(w, t, g)); + + } else if (Eq(t->op, "unlink")) { + Cleanup(gioop_unlink(w, t, g)); + + } else if (Eq(t->op, "unlinkat")) { + Cleanup(gioop_unlinkat(w, t, g)); + + } else if (Eq(t->op, "lstat")) { + Cleanup(gioop_lstat(w, t, g)); + + } else if (Eq(t->op, "fsync")) { + Cleanup(gioop_fsync(w, t, g)); + + } else if (Eq(t->op, "fdatasync")) { + Cleanup(gioop_fdatasync(w, t, g)); + + } else if (Eq(t->op, "sync")) { + Cleanup(gioop_sync(w, t, g)); + + } else if (Eq(t->op, "syncfs")) { + Cleanup(gioop_syncfs(w, t, g)); + + } else if (Eq(t->op, "sync_file_range")) { + Cleanup(gioop_sync_file_range(w, t, g)); + + } else if (Eq(t->op, "fcntl")) { + Cleanup(gioop_fcntl(w, t, g)); + + } else if (Eq(t->op, "fcntl")) { + Cleanup(gioop_fcntl(w, t, g)); + + } else if (Eq(t->op, "mmap2")) { + // Support for mmap added later + + } else if (Eq(t->op, "munmap")) { + // Support for mmap added later + + } else if (Eq(t->op, "mremap")) { + // Support for mmap added later + + } else if (Eq(t->op, "msync")) { + // Support for mmap added later + + } else if (Eq(t->op, "chmod")) { + Cleanup(gioop_chmod(w, t, g)); + + } else if (Eq(t->op, "fchmodat")) { + Cleanup(gioop_chmod(w, t, g)); + + } else if (Eq(t->op, "fchmod")) { + Cleanup(gioop_fchmod(w, t, g)); + + } else if (Eq(t->op, "chown")) { + Cleanup(gioop_chown(w, t, g)); + + } else if (Eq(t->op, "chown16")) { + Cleanup(gioop_chown(w, t, g)); + + } else if (Eq(t->op, "lchown")) { + Cleanup(gioop_lchown(w, t, g)); + + } else if (Eq(t->op, "lchown16")) { + Cleanup(gioop_lchown(w, t, g)); + + } else if (Eq(t->op, "fchown")) { + Cleanup(gioop_fchown(w, t, g)); + + } else if (Eq(t->op, "fchownat")) { + Cleanup(gioop_chown(w, t, g)); + + } else if (Eq(t->op, "exit_group")) { + Cleanup(gioop_exit_group(w, t, g)); + + } else { + Cleanup(ERROR;); + } + +cleanup: + +#ifdef LOG_FILTERED + if (ret != SUCCESS) + t->filtered_where = __FILE__; +#endif + + t->ret = ret; + return ret; +} + +status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd || t->path == NULL || t->flags == -1) { + return ERROR; + } + + gprocess_create_vfd_by_realfd(t->gprocess, t, g); + generate_vsize_by_path(g, t, NULL); + + Gioop_write(OPEN, "%ld|%s|%d|%d|open", + t->mapped_fd, t->path, t->mode, t->flags); + + if (t->fd > 0) + vsize_open(t->vsize, t->vfd, t->path, t->flags); + + return SUCCESS; +} + +status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd || t->path == NULL || t->flags == -1) { + return ERROR; + } + + gprocess_create_vfd_by_realfd(t->gprocess, t, g); + generate_vsize_by_path(g, t, NULL); + Gioop_write(OPEN_AT, "%ld|%s|%d|%d|openat", + t->mapped_fd,t->path, t->mode, t->flags); + if (t->fd > 0) + vsize_open(t->vsize, t->vfd, t->path, t->flags); + + return SUCCESS; +} + +status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd || t->path == NULL || t->flags == -1) { + return ERROR; + } + + gprocess_create_vfd_by_realfd(t->gprocess, t, g); + generate_vsize_by_path(g, t, NULL); + + Gioop_write(CREAT, "%ld|%s|%d|%d|creat", + t->mapped_fd, t->path, t->mode, t->flags); + if (t->fd > 0) + vsize_open(t->vsize, t->vfd, t->path, t->flags); + + return SUCCESS; +} + + +status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(CLOSE, "%ld|%d|close", t->mapped_fd, t->status); + + if (t->status == 0) + vsize_close(t->vsize, t->vfd); + + hmap_remove_l(t->gprocess->fd_map, t->fd); + hmap_remove_l(t->gprocess->vfd_map, t->mapped_fd); + + if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) + vfd_destroy(t->vfd); + + return SUCCESS; +} + +status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(STAT, "%s|%d|stat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(STATFS, "%s|%d|statfs", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(STATFS64, "%s|%d|statfs64", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSTAT, "%ld|%d|fstat", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(FSTAT_AT, "%s|%d|fstatat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSTATFS, "%ld|%d|fstatfs", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSTATFS64, "%ld|%d|fstatfs64", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL || t->path2 == NULL ) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(RENAME, "%s|%s|%d|rename", t->path, t->path2, t->status); + + if (t->status == 0) { + t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); + vsize_rename(t->vsize, t->vsize2, t->path, t->path2); + } + + return SUCCESS; +} + +status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL || t->path2 == NULL ) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(RENAME_AT, "%s|%s|%d|renameat", t->path, t->path2, t->status); + + if (t->status == 0) { + t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); + vsize_rename(t->vsize, t->vsize2, t->path, t->path2); + } + + return SUCCESS; +} +status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL || t->path2 == NULL ) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(RENAME_AT2, "%s|%s|%d|renameat2", + t->path, t->path2, t->status); + + if (t->status == 0) { + t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); + vsize_rename(t->vsize, t->vsize2, t->path, t->path2); + } + + return SUCCESS; +} + +status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READ, "%ld|%ld|read", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READ, "%ld|%ld|readv", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READAHEAD, "%ld|%ld|%ld|readahead", + t->mapped_fd, t->offset, t->count); + + return SUCCESS; +} + +status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READDIR, "%ld|%d|readdir", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(READLINK, "%s|%d|readlink", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(READLINK_AT, "%s|%d|readlinkat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(WRITE, "%ld|%ld|write", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_write(t->vsize, t->vfd, t->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(WRITEV, "%ld|%ld|writev", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_write(t->vsize, t->vfd, t->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(LSEEK, "%ld|%ld|%ld|%ld|lseek", + t->mapped_fd, t->offset, t->whence, t->bytes); + + if (t->bytes >= 0) + vsize_seek(t->vsize, t->vfd, t->bytes); + + return SUCCESS; +} + +status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(GETDENTS, "%ld|%ld|%ld|getdents", + t->mapped_fd, t->count, t->bytes); + + return SUCCESS; +} + +status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(MKDIR, "%s|%d|%d|mkdir", t->path, t->mode, t->status); + + if (t->status == 0) + vsize_mkdir(t->vsize, t->path); + + return SUCCESS; +} +status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(MKDIR, "%s|%d|rmdir", t->path, t->status); + + if (t->status == 0) + vsize_rmdir(t->vsize, t->path); + + return SUCCESS; +} +status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(MKDIR_AT, "%s|%d|%d|mkdirat", t->path, t->mode, t->status); + + if (t->status == 0) + vsize_mkdir(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(UNLINK, "%s|%d|unlink", t->path, t->status); + + if (t->status == 0) + vsize_unlink(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(UNLINK_AT, "%s|%d|unlinkat", t->path, t->status); + + if (t->status == 0) + vsize_unlink(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(LSTAT, "%s|%d|lstat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSYNC, "%ld|%d|fsync", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FDATASYNC, "%ld|%d|fdatasync", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g) +{ + Gioop_write(SYNC, "%d|sync", t->status); + + return SUCCESS; +} + +status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(SYNCFS, "%ld|%d|syncfs", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(SYNC_FILE_RANGE, "%ld|%ld|%ld|%d|sync_file_range", + t->mapped_fd, t->offset, t->bytes, t->status); + + return SUCCESS; +} + +status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + switch (t->F) { + case F_GETFD: + case F_GETFL: + case F_SETFD: + case F_SETFL: + break; + default: + return ERROR; + break; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FCNTL, "%ld|%d|%d|%d|fcntl", + t->mapped_fd, t->F, t->G, t->status); + + return SUCCESS; +} + +status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(CHMOD, "%s|%d|%d|chmod", t->path, t->mode, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FCHMOD, "%ld|%d|%d|fchmod", t->mapped_fd, t->mode, t->status); + + return SUCCESS; +} + +status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + // Hmm, maybe rename t->offset, because here it is used for the user UID + Gioop_write(CHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + // Hmm, maybe rename t->offset, because here it is used for the user UID + Gioop_write(FCHOWN, "%ld|%ld|%d|%d|fchown", t->mapped_fd, t->offset, t->G, t->status); + + return SUCCESS; +} + +status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + // Hmm, maybe rename t->offset, because here it is used for the user UID + Gioop_write(LCHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g) +{ + // It means that the process and all its threads terminate. + // Therefore close all file handles of that process! + hmap_run_cb2(t->gprocess->vfd_map, gioop_close_all_vfd_cb, t); + + // Remove virtual process from pid map and destroy it + amap_unset(g->pid_map, t->pid); + gprocess_destroy(t->gprocess); + + return SUCCESS; +} + +void gioop_close_all_vfd_cb(void *data, void *data2) +{ + gtask_s *t = data2; + t->vfd = data; + generate_s *g = t->generate; + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(CLOSE, "%ld|%d|close on exit_group", t->vfd->mapped_fd, 0); + vsize_close(t->vsize, t->vfd); +} + diff --git a/ioreplay/src/generate/gioop.h b/ioreplay/src/generate/gioop.h new file mode 100644 index 0000000..ad49713 --- /dev/null +++ b/ioreplay/src/generate/gioop.h @@ -0,0 +1,102 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GIOOP_H +#define GIOOP_H + +#include "../defaults.h" +#include "gwriter.h" +#include "gtask.h" +#include "generate.h" + + +// Helper macro regarding writing the .replay file! + +#define Gioop_write(op, ...) \ + fprintf(g->replay_fd, "%ld|%ld|%ld|0|0|%d|", \ + t->mapped_time, \ + (t->vsize ? t->vsize->id : 0),\ + t->gprocess->mapped_pid, \ + op); \ + fprintf(g->replay_fd, __VA_ARGS__); \ + fprintf(g->replay_fd, "@%ld", t->lineno); \ + fprintf(g->replay_fd, "|\n") + +/** + * @brief Function used when closing all virtual FDs of a virtual process + * + * This function is run on all virtual file handles whenever a virtual generate + * process object (gprocess_s) gets destroyed. This is on an exit_group + * syscall (a thread group, a process with all its threads, terminates). Upon + * process termination Linux also closes all its file descriptors! This is what + * we simulate here! + * + * @param data The pointer to the virtual file descriptor object + * @param data2 The pointer to the corresponding generate task object. + */ +void gioop_close_all_vfd_cb(void *data, void *data2); + +/** + * @brief Run a generate I/O operation on a given task + * + * @param w The writer object + * @param t The task object + * @return SUCCESS if everything went fine + */ +status_e gioop_run(gwriter_s *w, gtask_s *t); + +status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g); + +#endif // GIOOP_H diff --git a/ioreplay/src/generate/gparser.c b/ioreplay/src/generate/gparser.c new file mode 100644 index 0000000..514128f --- /dev/null +++ b/ioreplay/src/generate/gparser.c @@ -0,0 +1,356 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gparser.h" + +#include "gtask.h" +#include "gwriter.h" + +void* gparser_pthread_start(void *data) +{ + gparser_s *p = data; + generate_s *g = p->generate; + gwriter_s *w = g->writer; + gtask_s *t = NULL; + + do { + while (NULL != (t = rbuffer_get_next(p->queue))) { + // First extract + gparser_extract(p, t); + // Second, pass the task to the writer thread + rbuffer_insert(w->queue, t); + } + usleep(100); + } while (!p->terminate); + + while (NULL != (t = rbuffer_get_next(p->queue))) { + gparser_extract(p, t); + rbuffer_insert(w->queue, t); + } + + return NULL; +} + +gparser_s* gparser_new(generate_s *g) +{ + gparser_s *p = Malloc(gparser_s); + + p->generate = g; + p->terminate = false; + p->queue = rbuffer_new(1024); + + return p; +} + +void gparser_start(gparser_s *p) +{ + start_pthread(&p->pthread, gparser_pthread_start, (void*)p); +} + +void gparser_destroy(gparser_s *p) +{ + rbuffer_destroy(p->queue); + free(p); +} + +void gparser_terminate(gparser_s *p) +{ + p->terminate = true; + pthread_join(p->pthread, NULL); +} + +void gparser_extract(gparser_s *p, gtask_s *t) +{ + status_e ret = SUCCESS; + generate_s *g = p->generate; + + char *saveptr; + char* tok = strtok2_r(t->line, ";:,", &saveptr); + int ntoks = 0; + + while (tok) { + if (++ntoks > MAX_TOKENS) { + ret = ERROR; + break; + } + ret = gparser_extract_tok(p, t, tok); + if (ret != SUCCESS) + break; + + tok = strtok2_r(NULL, ";:,", &saveptr); + } + + if (ret == SUCCESS) { + + // Check for the existance of mandatory values! + if (t->pid < 0 || t->tid < 0) { + Cleanup(ERROR); + + } else if (t->op == NULL) { + Cleanup(ERROR); + + } else if (t->mapped_time == -1) { + Cleanup(ERROR); + } + + // We are inserting ".ioreplay/NAME" to the paths. This enables us to + // run multiple tests simoultaneously. + + if (t->path) { + if (!mounts_transform_path(g->mps, g->name, + t->path, &t->path_r)) { + Cleanup(ERROR); + } + if (t->path_r) + t->path = t->path_r; + } + + if (t->path2) { + if (!mounts_transform_path(g->mps, g->name, + t->path2, &t->path2_r)) { + Cleanup(ERROR); + } + if (t->path2_r) + t->path2 = t->path2_r; + } + + } + +cleanup: + + t->ret = ret; + +#ifdef LOG_FILTERED + t->filtered_where = __FILE__; +#endif +} + +status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok) +{ + status_e ret = SUCCESS; + + if (gparser_token_not_ok(p, tok)) { + Cleanup(ERROR); + } + + generate_s *g = t->generate; + + char key = tok[0]; + char *value = tok; + value += 2; + + switch (key) { + case 'a': + // Address + t->address = strtol(value, NULL, 10); + break; + + case 'A': + // Address 2 + t->address2 = strtol(value, NULL, 10); + break; + + case 'b': + // Bytes + if (t->bytes != -1) { + Cleanup(ERROR); + } + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->bytes = strtol(value, NULL, 10); + break; + + case 'c': + // Count + if (t->count != -1) { + Cleanup(ERROR); + } + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->count = strtol(value, NULL, 10); + break; + + case 'd': + // Descriptor + if (t->fd != -1) { + Cleanup(ERROR); + } + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->fd = atoi(value); + if (t->fd > 0) + t->has_fd = true; + break; + + case 'f': + // Flags + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->flags = atoi(value); + break; + + case 'i': + // PID:TID + t->pidtid = value; + // Extract PID and TID from "PID:TID" + if (!gparser_get_pidtid(p, t->pidtid, &t->pid, &t->tid)) { + Cleanup(ERROR); + } + break; + + case 'm': + // Mode + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->mode = atoi(value); + break; + + case 'o': + // Operation + t->op = value; + break; + + case 'O': + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->offset = strtol(value, NULL, 10); + break; + + case 'W': + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->whence = strtol(value, NULL, 10); + break; + + case 'p': + // File path + t->path = value; + chreplace(t->path, '|', '_'); + strunquote(t->path); + break; + + case 'P': + // File path 2 + t->path2 = value; + chreplace(t->path2, '|', '_'); + strunquote(t->path2); + break; + + case 's': + // Cleanup status + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->status = atoi(value); + break; + + case 't': + // Time + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->mapped_time = strtol(value, NULL, 10); + // Start replay time from 0 + if (g->start_time == -1) { + g->start_time = t->mapped_time; + } + t->mapped_time -= g->start_time; + break; + + case 'F': + // FCNTL function + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->F = atoi(value); + break; + + case 'G': + // FCNTL argument + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->G = atoi(value); + break; + + case 'T': + break; + + default: + // Unknown key + { + Cleanup(ERROR); + } + } + +cleanup: + if (t->path_r) { + free(t->path_r); + t->path_r = NULL; + } + if (t->path2_r) { + free(t->path2_r); + t->path2_r = NULL; + } + + return ret; +} + +bool gparser_token_not_ok(gparser_s *p, char *tok) +{ + if (strlen(tok) < 3) { + return true; + + } else if (tok[1] != '=') { + return true; + } + + return false; +} + +bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid) +{ + char *pos = strchr(pidtid, ':'); + + if (pos) { + char *tmp = pos; + tmp++; + + if (is_number(tmp)) { + *tid = atol(tmp); + } else { + return false; + } + + pos[0] = '\0'; + if (is_number(pidtid)) { + *pid = atol(pidtid); + } else { + return false; + } + } + + else { + return false; + } + + return (*pid >= 0 && *tid >= 0); +} diff --git a/ioreplay/src/generate/gparser.h b/ioreplay/src/generate/gparser.h new file mode 100644 index 0000000..f3e204a --- /dev/null +++ b/ioreplay/src/generate/gparser.h @@ -0,0 +1,113 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GPARSER_H +#define GPARSER_H + +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "generate.h" +#include "gtask.h" + +/** + * @brief The parser definition + * + * The parser is to extract all information from the .capture file. + */ +typedef struct gparser_s_ { + bool terminate; /**< The parser thread will terminate if set to true */ + generate_s *generate; /**< The generate object */ + pthread_t pthread; /**< The posix thread */ + rbuffer_s *queue; /**< A queue of task objects */ +} gparser_s; + +/** + * @brief Creates a new parser + * + * @param g The generate object + * @return The new parser object + */ +gparser_s* gparser_new(generate_s *g); + +/** + * @brief Starts the parser thread + * + * @param p The parser object + */ +void gparser_start(gparser_s *p); + +/** + * @brief Terminates the parser thread + * + * @param p The parser object + */ +void gparser_terminate(gparser_s *p); + +/** + * @brief Destroys the parser thread + * + * @param p The parser object + */ +void gparser_destroy(gparser_s *p); + +/** + * @brief Extracts information a .capture line + * + * Extracts information from a .capture line and stores it into the task + * object. + * + * @param p The parser object + * @param t The task object + */ +void gparser_extract(gparser_s *p, gtask_s *t); + +/** + * @brief Extracts information from a specific token string + * + * @param p The parser object + * @param t The task object + * @param tok The token string + * @return Returns with SUCCESS on success + */ +status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok); + +/** + * @brief Verifies the correctness of a token + * + * @param p The parser object + * @param tok The token to be verified + * @return true if token verified successfully + */ +bool gparser_token_not_ok(gparser_s *p, char *tok); + +/** + * @brief Checks whether the pidtid string is correct or not + * + * @param p The parser object + * @param pidtid The string to check + * @param pid The pointer to the resulting pid + * @param tid The pointer to the resulting tid + * @return true on success + */ +bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid); + +/** + * @brief Entry point of the parser POSIX thread + * + * @param data A pointer to the parser object + * return Always NULL + */ +void* gparser_pthread_start(void *data); + +#endif // GPARSER_H diff --git a/ioreplay/src/generate/gprocess.c b/ioreplay/src/generate/gprocess.c new file mode 100644 index 0000000..6a0b37a --- /dev/null +++ b/ioreplay/src/generate/gprocess.c @@ -0,0 +1,101 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gprocess.h" + +#include "../vfd.h" +#include "gioop.h" + +void _gprocess_vfd_map_destroy_cb(void *data) +{ + vfd_destroy(data); +} + +gprocess_s* gprocess_new(const long pid, const long mapped_pid) +{ + gprocess_s* gp = Malloc(gprocess_s); + + gp->pid = pid; + gp->mapped_pid = mapped_pid; + gp->max_mapped_fd = 0; + gp->fd_map = hmap_new_l(1024); + gp->vfd_map = hmap_new_l(1024); + gp->vfd_map->data_destroy = _gprocess_vfd_map_destroy_cb; + + return gp; +} + +void gprocess_destroy(gprocess_s *gp) +{ + hmap_destroy(gp->vfd_map); + hmap_destroy(gp->fd_map); + free(gp); +} + +void gprocess_create_vfd_by_realfd(gprocess_s *gp, gtask_s *t, generate_s *g) +{ + if (t->fd < 0) + return; + + // Check whether the real FD is still open according to the .capture log + long old_mapped = (long) hmap_get_l(gp->fd_map, t->fd); + if (old_mapped) { + + // That real file descriptor is already with a mapping to a virtual + // file descriptor. This may happen when SystemTap missed to trace a + // 'close' syscall. We are inserting a close now... + + t->vfd = hmap_get_l(gp->vfd_map, old_mapped); + + hmap_remove_l(gp->fd_map, t->fd); + hmap_remove_l(gp->vfd_map, old_mapped); + + if (t->vfd) { + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(CLOSE, "%ld|%d|close inserted", old_mapped, 0); + vsize_close(t->vsize, t->vfd); + if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) + vfd_destroy(t->vfd); + } + } + + t->vfd = rbuffer_get_next(g->vfd_buffer); + t->mapped_fd = ++g->num_mapped_fds; + if (!t->vfd) + t->vfd = vfd_new(t->fd, t->mapped_fd, t->path); + else + vfd_update(t->vfd, t->fd, t->mapped_fd, t->path); + t->vfd->free_path = t->path_r != NULL; + + hmap_insert_l(gp->vfd_map, t->mapped_fd, t->vfd); + hmap_insert_l(gp->fd_map, t->fd, (void*)t->mapped_fd); +} + +status_e gprocess_vfd_by_realfd(gprocess_s *gp, gtask_s *t) +{ + t->mapped_fd = (long) hmap_get_l(gp->fd_map, t->fd); + if (t->mapped_fd == 0) { + // No corresponding virtual fd number mapping + t->has_fd = false; + + } else { + t->vfd = hmap_get_l(gp->vfd_map, t->mapped_fd); + if (!t->vfd) { + return ERROR; + } + t->mapped_fd = t->vfd->mapped_fd; + } + + return SUCCESS; +} diff --git a/ioreplay/src/generate/gprocess.h b/ioreplay/src/generate/gprocess.h new file mode 100644 index 0000000..47e5037 --- /dev/null +++ b/ioreplay/src/generate/gprocess.h @@ -0,0 +1,90 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GPROCESS_H +#define GPROCESS_H + +#include "../datas/hmap.h" +#include "../defaults.h" +#include "gtask.h" +#include "generate.h" + +// Forward declarations (header include hell) +struct gtask_s_; +struct generate_s_; + +/** + * @brief Virtual process object used for generating .replay file + * + * An object of this represents a Linux process in generate context. + * In Linux every process owns * its own file descriptor table which is + * simulated here. Usually, a Linux process re-uses a FD number once not used + * anymore (e.g. after a close). However, as we want to increase concurrency + * while replaying the I/O we want * to ensure to always use unique file + * descriptor IDs for every open. Thats why we use max_mapped_fd to always + * map a real FD number to a uniq virtual FD number. + */ +typedef struct gprocess_s_ { + long pid; /**< The real PID */ + long mapped_pid; /**< The mapped PID */ + hmap_s *vfd_map; /**< All virtual file descriptors of that process */ + hmap_s *fd_map; /**< All mappings from real fd to virtual fd */ + long max_mapped_fd; /**< The max mapped fd number */ +} gprocess_s; + +/** + * @brief Creates a new gprocess object + * + * @param pid The process ID + * @param mapped_pid the mapped PID + * @return The new gprocess object + */ +gprocess_s* gprocess_new(const long pid, const long mapped_pid); + +/** + * @brief Destroys a gprocess object + * + * @param gp The gprocess object + */ +void gprocess_destroy(gprocess_s *gp); + +/** + * @brief Creates a new virtual FD from a given real FD number + * + * In ioreplay we map the real file descriptor (the fd number protocolled in + * the.capture file) to a virtual file descriptor (the fd numner written to the + * .replay file). The purpose is to increase concurrency of the I/O during + * replay. Normally, a process would reuse the same file descriptor number + * once closed earlier. However, when replaying we can't reuse the number if + * we want to replay the I/O on multiple paths in parallel. Therefore, it is + * ensured that the virtual file descriptor number in the .replay file is + * always * unique for every open! + * + * @param gp The process object + * @param t The task object (the vfd pointer will be stored to * t->vfd) + * @param g The generate object + */ +void gprocess_create_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t, + struct generate_s_ *g); + +/** + * @brief Retrieves a virtual FD from a given real FD number + * + * @param gp The process object + * @param t The task object (the vfd pointer will be stored to * t->vfd) + * @return SUCCESS if everything went smothly! + */ +status_e gprocess_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t); + +#endif // GPROCESS_H diff --git a/ioreplay/src/generate/gtask.c b/ioreplay/src/generate/gtask.c new file mode 100644 index 0000000..55a1124 --- /dev/null +++ b/ioreplay/src/generate/gtask.c @@ -0,0 +1,91 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtask.h" + +gtask_s* gtask_new(void *generate) +{ + gtask_s *t = Malloc(gtask_s); + + t->generate = generate; + t->line = NULL; + t->path_r = NULL; + t->path2_r = NULL; +#ifdef LOG_FILTERED + t->original_line = NULL; +#endif + + return t; +} + +void gtask_init(gtask_s *t, char *line, const unsigned long lineno) +{ + if (t->line) + free(t->line); + t->line = Clone(line); + + if (t->path_r) + free(t->path_r); + if (t->path2_r) + free(t->path2_r); + +#ifdef LOG_FILTERED + if (t->original_line) + free(t->original_line); + t->original_line = Clone(line); + t->filtered_where = NULL; +#endif + + t->bytes = -1; + t->address = 0; + t->address2 = 0; + t->count = -1; + t->F = -1; + t->fd = -1; + t->flags = -1; + t->G = -1; + t->has_fd = false; + t->vsize = NULL; + t->vsize2 = NULL; + t->lineno = lineno; + t->mapped_fd = -1; + t->mapped_time = -1; + t->mode = -1; + t->offset = -1; + t->op = NULL; + t->path2 = NULL; + t->path2_r = NULL; + t->path = NULL; + t->path_r = NULL; + t->gprocess = NULL; + t->pid = -1; + t->pidtid = NULL; + t->ret = 0; + t->status = -1; + t->tid = -1; + t->vfd = NULL; + t->whence = -1; +} + +void gtask_destroy(gtask_s *t) +{ + if (t->line) + free(t->line); + if (t->path_r) + free(t->path_r); + if (t->path2_r) + free(t->path2_r); + free(t); +} + diff --git a/ioreplay/src/generate/gtask.h b/ioreplay/src/generate/gtask.h new file mode 100644 index 0000000..2f364d3 --- /dev/null +++ b/ioreplay/src/generate/gtask.h @@ -0,0 +1,100 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GTASK_H +#define GTASK_H + +#include "vsize.h" + +#include "gprocess.h" +#include "../vfd.h" +#include "../datas/amap.h" +#include "../datas/hmap.h" +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "../mounts.h" +#include "../options.h" + +/** + * @brief The generate task definition + * + * The gtask holds all possible variables required to process a particular + * .capture line and to generate the corresponding .replay line + */ +typedef struct gtask_s_ { + bool has_fd; /**< True if task has a file descriptor number */ + char *line; /**< A pointer to the remaining part of the .capture line */ + char *op; /**< Operation/syscall name */ + char *path2; /**< A second path name (e.g. for rename) */ + char *path2_r; /**< Work around to track mallocs, so it can be freed */ + char *path; /**< Path name */ + char *path_r; /**< Work around to track mallocs, so it can be freed */ + char *pidtid; /**< String representing pid:tid */ + int F; /**< Arguments for fcntl syscall */ + int G; /**< Arguments for fcntl syscall */ + int fd; /**< File descriptor number */ + int flags; /**< File open flags */ + int mode; /**< File open mode */ + int ret; /**< ioreplay process status, SUCCESS if everything is alright */ + int status; /**< Operation/syscall return status */ + long address2; /**< Another address (used by mmap related syscalls) */ + long address; /**< An address (used by mmap related syscalls) */ + long bytes; /**< Amount of bytes */ + long count; /**< A count */ + long lineno; /**< The current line number */ + long mapped_fd; /**< The mapped file descriptor number */ + long mapped_time; /**< The mapped time */ + long offset; /**< A offset */ + long pid; /**< The process ID */ + long tid; /**< The thread ID */ + long whence; /**< Whence */ + vfd_s *vfd; /**< A pointer to the virtual file descriptor */ + struct gprocess_s_ *gprocess; /**< A pointer to the process object */ + void *generate; /**< A pointer to the generate object */ + vsize_s *vsize2; /**< Pointer to a second virtual size object */ + vsize_s *vsize; /**< Pointer to the virtual size object */ +#ifdef LOG_FILTERED + char *original_line; /**< Only used for debugging purposes */ + char *filtered_where; /**< Only used for debugging purposes */ +#endif +} gtask_s; + +/** + * @brief Creates a new task object + * + * @param generate A pointer to the generate object + * @return The new task object + */ +gtask_s* gtask_new(void *generate); + +/** + * @brief Initialises a taks object + * + * This function is used in particular when we recycle/reuse an old + * gtask object. + * + * @param t The gtask object + * @param line The corresponding line from the .capture file + * @param lineno The line number + */ +void gtask_init(gtask_s *t, char *line, const unsigned long lineno); + +/** + * @brief Destroys a given task object + * + * @param t The task object + */ +void gtask_destroy(gtask_s *t); + +#endif // GTASK_H diff --git a/ioreplay/src/generate/gwriter.c b/ioreplay/src/generate/gwriter.c new file mode 100644 index 0000000..e0d448e --- /dev/null +++ b/ioreplay/src/generate/gwriter.c @@ -0,0 +1,85 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gwriter.h" + +#include "gtask.h" +#include "generate.h" +#include "gioop.h" +#include "../opcodes.h" + +void* gwriter_pthread_start(void *data) +{ + gwriter_s *w = data; + generate_s *g = w->generate; + gtask_s *t = NULL; + + do { + while (NULL != (t = rbuffer_get_next(w->queue))) { +#ifdef LOG_FILTERED + // Logging filtered lines + if (SUCCESS != gioop_run(w, t)) { + fprintf(g->replay_fd, "#FILTERED @%ld %s", t->lineno, + t->original_line); + } +#else + gioop_run(w, t); +#endif + rbuffer_insert(g->reuse_queue, t); + } + usleep(100); + } while (!w->terminate); + + while (NULL != (t = rbuffer_get_next(w->queue))) { +#ifdef LOG_FILTERED + if (SUCCESS != gioop_run(w, t)) { + fprintf(g->replay_fd, "#FILTERED @%ld %s\n", t->lineno, + t->original_line); + } +#else + gioop_run(w, t); +#endif + rbuffer_insert(g->reuse_queue, t); + } + + return NULL; +} + +gwriter_s* gwriter_new(generate_s *g) +{ + gwriter_s *w = Malloc(gwriter_s); + + w->generate = g; + w->terminate = false; + w->queue = rbuffer_new(1024); + + return w; +} + +void gwriter_start(gwriter_s *w) +{ + start_pthread(&w->pthread, gwriter_pthread_start, (void*)w); +} + +void gwriter_destroy(gwriter_s *w) +{ + rbuffer_destroy(w->queue); + free(w); +} + +void gwriter_terminate(gwriter_s *w) +{ + w->terminate = true; + pthread_join(w->pthread, NULL); +} diff --git a/ioreplay/src/generate/gwriter.h b/ioreplay/src/generate/gwriter.h new file mode 100644 index 0000000..4295580 --- /dev/null +++ b/ioreplay/src/generate/gwriter.h @@ -0,0 +1,86 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GWRITER_H +#define GWRITER_H + +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "vsize.h" +#include "gtask.h" +#include "generate.h" + +// Forward declaration (header include hell) +struct gtask_s_; +struct generate_s_; + +/** + * @brief Definition of the writer object + * + * The writer utilises the information extracted by the parser to actually + * write the .replay file. + */ +typedef struct gwriter_s_ { + bool terminate; /**< The writer thread will terminate if set to true */ + struct generate_s_ *generate; /**< The generate object */ + pthread_t pthread; /**< The posix thread */ + rbuffer_s *queue; /**< A queue of task objects */ +} gwriter_s; + +/** + * @brief Creates a new writer + * + * @param g The generate object + * @return The new writer object + */ +gwriter_s* gwriter_new(struct generate_s_ *g); + +/** + * @brief Starts the writer thread + * + * @param w The writer object + */ +void gwriter_start(gwriter_s *w); + +/** + * @brief Terminates the writer thread + * + * @param w The writer object + */ +void gwriter_terminate(gwriter_s *w); + +/** + * @brief Destroys the writer thread + * + * @param w The writer object + */ +void gwriter_destroy(gwriter_s *w); + +/** + * @brief Writes a line to the .replay file + * + * @param w The writer object + * @param t The task object + */ +void gwriter_write(gwriter_s *w, struct gtask_s_ *t); + +/** + * @brief Entry function of the writer pthread + * + * @param data A pointer to the writer object + * @return Always returns a NULL pointer if it doesnt crash! + */ +void* gwriter_pthread_start(void *data); + +#endif // GWRITER_H diff --git a/ioreplay/src/generate/vsize.c b/ioreplay/src/generate/vsize.c new file mode 100644 index 0000000..f2d56ba --- /dev/null +++ b/ioreplay/src/generate/vsize.c @@ -0,0 +1,247 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "vsize.h" + +#include "generate.h" + +// Helper macros + +#define _Set_file(v) v->is_file = true; v->unsure = v->is_dir = false +#define _Set_dir(v) v->is_dir = true; v->unsure = v->is_file = false +#define _Set_unsure(v) v->unsure = true +#define _Set_inserted(v) v->inserted = true +#define _Set_renamed(v) v->renamed = true +#define _Set_required(v) v->required = true + +vsize_s* vsize_new(char *file_path, const unsigned long id, + void *generate) +{ + vsize_s *v = Malloc(vsize_s); + + v->generate = generate; + v->id = id; + v->inserted = false; + v->is_dir = false; + v->is_file = false; + v->offset = -1; + v->path = Clone(file_path); + v->renamed = false; + v->required = false; + v->unsure = false; + v->updates = 0; + v->vsize = 0; + v->vsize_deficit = 0; + + return v; +} + +void vsize_destroy(vsize_s *v) +{ + if (!v) + return; + + free(v->path); + free(v); +} + +void init_parent_dir(vsize_s *v, const char *path) +{ + generate_s *g = v->generate; + char *clone = Clone(path); + char *parent = dirname(clone); + + vsize_s *v_parent = hmap_get(g->vsize_map, parent); + if (!v_parent) { + + // Parent directory does not yet have a vsize! + // Create a vsize object for it and set it as a pre-requirement + // so that the directory can be created during init mode. + + v_parent = vsize_new(parent, ++g->num_vsizes, g); + hmap_insert(g->vsize_map, parent, v_parent); + + _Set_required(v_parent); + _Set_dir(v_parent); + + // This is for debugging purposes only + _Set_inserted(v_parent); + v_parent->updates++; + + } else if (v_parent->unsure) { + // We now know for sure that this path must be a directory! + _Set_dir(v_parent); + v_parent->updates++; + } + + free(clone); +} + +void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags) +{ + + // v->first_encounter == false means, that this is the first occurance of + // this path and we didn't initialise it (means we didn't ensure that + // we want to create all parent directories etc. + + if (v->updates == 0) { + // We may use a recycled vfd object! When opening a file we always + // assume that the offset is 0! + vfd_s *vfd_ = vfd; + vfd_->offset = 0; + init_parent_dir(v, path); + + if (Has(flags, O_DIRECTORY)) { + _Set_required(v); + _Set_dir(v); + + } else if (Hasnt(flags, O_CREAT)) { + _Set_required(v); + _Set_file(v); + _Set_unsure(v); + } + v->updates++; + + } else if (v->unsure) { + if (Has(flags, O_DIRECTORY)) { + // Now we know for sure that this path must be a directory! + _Set_dir(v); + v->updates++; + } + } +} + +void vsize_close(vsize_s *v, void* vfd) +{ + vfd_s *vfd_ = vfd; + vfd_->offset = 0; + v->updates++; +} + +void vsize_stat(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + _Set_file(v); + + // We are not 100% sure that this is really a file, + // the path might be still a directory though! + _Set_unsure(v); + v->updates++; + } +} + +void vsize_rename(vsize_s *v, vsize_s *v2, + const char *path, const char *path2) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + _Set_file(v); + _Set_unsure(v); + v->updates++; + } + + if (v2->updates == 0) { + init_parent_dir(v2, path2); + _Set_file(v2); + + // We are not 100% sure that this is really a file, + // the path might be still a directory though! + _Set_unsure(v2); + + // For debugging purposes only + _Set_renamed(v2); + v2->updates++; + } +} + +void vsize_adjust(vsize_s *v, vfd_s* vfd) +{ + if (v->vsize >= vfd->offset) + return; + + long deficit = v->vsize - vfd->offset; + if (deficit < v->vsize_deficit) { + v->vsize_deficit = deficit; + _Set_required(v); + _Set_file(v); + } +} + +void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes) +{ + vfd_s *vfd_ = vfd; + vfd_->offset += bytes; + vsize_adjust(v, vfd_); + v->updates++; +} + +void vsize_seek(vsize_s *v, void *vfd, const long new_offset) +{ + //vfd_s *vfd_ = vfd; + + // The file's offset can be greater than the file's current size, in which + // case the next write to the file will extend the file. This is referred + // to as creating a hole in a file and is allowed. However, this behaviour + // does not suit the estimation of the file size before we want to run the + // test. + + // TODO: Implement file hole support! + //v->updates++; +} + +void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes) +{ + vfd_s *vfd_ = vfd; + vfd_->offset += bytes; + + if (v->vsize < vfd_->offset) + v->vsize = vfd_->offset; + + v->updates++; +} + +void vsize_mkdir(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_dir(v); + v->updates++; + } +} + +void vsize_rmdir(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + _Set_dir(v); + v->updates++; + } +} + +void vsize_unlink(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + if (!v->is_dir) { + _Set_file(v); + _Set_unsure(v); + } + v->updates++; + } +} diff --git a/ioreplay/src/generate/vsize.h b/ioreplay/src/generate/vsize.h new file mode 100644 index 0000000..bb1008e --- /dev/null +++ b/ioreplay/src/generate/vsize.h @@ -0,0 +1,180 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef VSIZE_H +#define VSIZE_H + +#include "../utils/utils.h" +#include "../datas/hmap.h" +#include "../vfd.h" + +/** + * @brief Definition of a virtual size object + * + * The virtual size is used to determine the expected type and size of a file. + * This piece of information will be added to the INIT section of the the + * .replay file. That file then will be created during test initialisation. + * before running the test. It is very likely the case that the test requires + * a file of a certain size already to be present, so it can be read from disk. + */ +typedef struct vsize_s_ { + char *path; /**< The path to the file/directory */ + off_t offset; /**< The current file offset */ + unsigned long id; /**< The vsize id */ + void *generate; /**< A pointer to the generate object */ + long vsize; /**< The virtual size */ + long vsize_deficit; /**< Size to use for file creating during init mode */ + bool renamed; /**< True if file/dir has been renamed */ + bool required; /**< True if init mode will create this file/dir */ + bool is_dir; /**< True if this file/dir is a directory */ + bool is_file; /**< True if this file/dir is a regular file */ + bool unsure; /**< True if the file type is not fully clear */ + long updates; /**< Amount of times this vsize has been updated */ + bool inserted; /**< For debugging purposes only */ +} vsize_s; + +/** + * @brief Creates a new vsize object + * + * @param file_path The corresponding file path + * @param id The vsize vsize aka ID + * @param generate The generate object + * @return The new vsize object + */ +vsize_s* vsize_new(char *file_path, const unsigned long id, void *generate); + +/** + * @brief Destroys a vsize object + * + * @param v The vsize object + */ +void vsize_destroy(vsize_s *v); + +/** + * @brief Ensures that the parent directory exists + * + * This function ensures that the parent directory exists as a vsize object! + * + * @param v The vsize object + * @param path The given path + */ +void init_parent_dir(vsize_s *v, const char *path); + +/** + * @brief Adjusts the vsize + * + * Compares the virtual file size of the file in the vsize + * object to the the offset in the virtual file descriptor. + * In case the offset is higher we have a size deficit and + * we need to mark it. That way ioreplay can ensure that + * during init mode it will create a file with the correct + * size prior of running the test! + * + * @param v The virtual size object + * @param vfd The virtual file descriptor object + */ +void vsize_adjust(vsize_s *v, vfd_s* vfd); + +/** + * @brief Adjust vsize on open + * + * @param v The virtual size object + * @param vfd The virtual file descriptor object + * @param path The file open path + * @param flags The file open flags + */ +void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags); + +/** + * @brief Adjust vsize on close + * + * @param v The virtual size object + * @param vfd The virtual file descriptor object + */ +void vsize_close(vsize_s *v, void *vfd); + +/** + * @brief Adjust vsize on stat + * + * @param v The virtual size object + * @param path The stat path + */ +void vsize_stat(vsize_s *v, const char *path); + +/** + * @brief Adjust vsize on rename + * + * @param v The virtual size object + * @param v2 The virtual size object of path2 + * @param path The first file path + * @param path2 The second file path + */ +void vsize_rename(vsize_s *v, vsize_s *v2, + const char *path, const char *path2); + +/** + * @brief Adjust vsize on read + * + * @param v The virtual size object + * @param vfd The virtual vile descriptor object + * @param path The file path + * @param bytes The amount of bytes read + */ +void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes); + +/** + * @brief Adjust vsize on seek + * + * @param v The virtual size object + * @param vfd The virtual vile descriptor object + * @param new_offset The new file offset after seek + */ +void vsize_seek(vsize_s *v, void *vfd, const long new_offset); + +/** + * @brief Adjust vsize on write + * + * @param v The virtual size object + * @param vfd The virtual vile descriptor object + * @param path The file path + * @param bytes The amount of bytes written + */ +void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes); + +/** + * @brief Adjust vsize on mkdir + * + * @param v The virtual size object + * @param path The directory path + */ +void vsize_mkdir(vsize_s *v, const char *path); + +/** + * @brief Adjust vsize on rmdir + * + * @param v The virtual size object + * @param path The directory path + */ +void vsize_rmdir(vsize_s *v, const char *path); + +/** + * @brief Adjust vsize on unlink + * + * @param v The virtual size object + * @param path The file path + */ +void vsize_unlink(vsize_s *v, const char *path); + +#endif // VSIZE_H + diff --git a/ioreplay/src/init/init.c b/ioreplay/src/init/init.c new file mode 100644 index 0000000..988729e --- /dev/null +++ b/ioreplay/src/init/init.c @@ -0,0 +1,226 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "init.h" + +#include "../datas/stack.h" +#include "itask.h" +#include "ithread.h" +#include "../meta/meta.h" +#include "../mounts.h" +#include "../utils/futils.h" + + +init_s *init_new(options_s *opts) +{ + init_s *i = Malloc(init_s); + + i->opts = opts; + i->mounts = mounts_new(opts); + i->threads_map = amap_new(i->mounts->count); + i->reuse_queue = rbuffer_new(4096); + i->replay_fd = Fopen(opts->replay_file, "r"); + + pthread_mutex_init(&i->reuse_queue_mutex, NULL); + + return i; +} + +void init_destroy(init_s *i) +{ + amap_destroy(i->threads_map); + mounts_destroy(i->mounts); + + itask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(i->reuse_queue))) { + itask_destroy(task); + } + rbuffer_destroy(i->reuse_queue); + + fclose(i->replay_fd); + pthread_mutex_destroy(&i->reuse_queue_mutex); + + free(i); +} + +void init_extract_header(init_s *i, off_t *init_offset) +{ + options_s *opts = i->opts; + meta_s *m = meta_new(i->replay_fd); + meta_read_start(m); + + long version = 0; + if (meta_read_l(m, "version", &version)) { + Put("Replay version is '%ld'", version); + if (version != REPLAY_VERSION) { + Error(".replay file of incompatible version, got %x, expected %x", + (int)version, REPLAY_VERSION); + } + } + + char *user; + if (meta_read_s(m, "user", &user)) { + Put("Setting user to '%s'", user); + opts->user = user; + } + + char *name; + if (meta_read_s(m, "name", &name)) { + Put("Setting name to '%s'", name); + opts->name = name; + } + + if (meta_read_l(m, "init_offset", init_offset)) { + if (*init_offset < 0) { + Error("Offset overflow (init offset too large in .replay)"); + } + Put("Setting init offset to '%ld'", *init_offset); + } + + meta_destroy(m); +} + +status_e init_run(options_s *opts) +{ + status_e ret = SUCCESS; + init_s *i = init_new(opts); + + off_t init_offset; + init_extract_header(i, &init_offset); + + // Ensure that all ./replay/NAME directories exist + mounts_init(i->mounts); + + // Don't do messy stuff as super user + drop_root(opts->user); + + // We need to clean up garbish from previous runs! + if (opts->purge) + mounts_purge(i->mounts); + else + mounts_trash(i->mounts); + + Out("Creating all files and directories requried for test '%s'...", + opts->name); + + // Seek to the INIT section + fseeko(i->replay_fd, init_offset, SEEK_SET); + + bool is_file = false, is_dir = false; + long vsize = 0; + char *path; + + // Stats + long dirs_created = 0; + long files_created = 0; + long files_total_size = 0; + + // Helper variables for getline + char *line = NULL; + size_t len = 0, read = 0; + char *saveptr; + + stack_s *all_threads = stack_new(); + + // Process the INIT section of the .replay file line by line. + + while ((read = getline(&line, &len, i->replay_fd)) != -1) { + char *tok = strtok_r(line, "|", &saveptr); + + for (int ntok = 0; tok; ntok++) { + switch (ntok) { + case 0: + is_dir = atoi(tok) == 1; + break; + case 1: + is_file = atoi(tok) == 1; + break; + case 2: + vsize = atol(tok); + if (vsize < 0) { + Error("Size overflow"); + } + break; + case 3: + path = tok; + break; + default: + break; + } + + tok = strtok_r(NULL, "|", &saveptr); + } + + itask_s *task = rbuffer_get_next(i->reuse_queue); + + if (!task) { + task = itask_new(); + + } else { + itask_extract_stats(task, &dirs_created, &files_created, + &files_total_size); + } + + // Set new task values + if (is_dir) { + task->is_dir = true; + + } else if (is_file) { + task->is_file = true; + task->vsize = vsize; + } + task->path = Clone(path); + + // We run one init thread per mount point + int mnr = mounts_get_mountnumber(i->mounts, path); + ithread_s *t = amap_get(i->threads_map, mnr); + + if (!t) { + t = ithread_new(i); + amap_set(i->threads_map, mnr, t); + stack_push(all_threads, t); + ithread_start(t); + } + + //itask_print(task); + while (!rbuffer_insert(t->queue, task)) + usleep(1000); + } + + ithread_s *t = NULL; + while (NULL != (t = stack_pop(all_threads))) { + ithread_terminate(t); + ithread_destroy(t); + } + stack_destroy(all_threads); + + itask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(i->reuse_queue))) { + itask_extract_stats(task, &dirs_created, &files_created, + &files_total_size); + itask_destroy(task); + } + + Put("Done!"); + + Put("Created %ld files (net total size: %.2fg) and %ld directories!", + files_created, files_total_size/(1024*1024*1024.0), + dirs_created); + + init_destroy(i); + + Put("You are ready to fire up the test now"); + + return ret; +} diff --git a/ioreplay/src/init/init.h b/ioreplay/src/init/init.h new file mode 100644 index 0000000..3d9f9e9 --- /dev/null +++ b/ioreplay/src/init/init.h @@ -0,0 +1,64 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INIT_H +#define INIT_H + +#include "../defaults.h" +#include "../options.h" +#include "../datas/amap.h" +#include "../datas/rbuffer.h" +#include "../mounts.h" + +typedef struct init_s_ { + amap_s *threads_map; + rbuffer_s *reuse_queue; + options_s *opts; + mounts_s *mounts; + FILE *replay_fd; + pthread_mutex_t reuse_queue_mutex; +} init_s; + +/** + * @brief Creates a new init object + * + * @param opts The options object + * @return The new mounts object + */ +init_s* init_new(options_s *opts); + +/** + * @brief Destroys the init object + * + * @param i The init object + */ +void init_destroy(init_s *i); + +/** + * @brief Initialises the test environment + * + * @param opts The options object + * @return SUCCESS if initialised without any issues + */ +status_e init_run(options_s *opts); + +/** + * @brief Extracts some useful information from the .replay meta header + * + * @param i The init object + * @param init_offset To store the offset of the init section + */ +void init_extract_header(init_s *i, off_t *init_offset); + +#endif // INIT_H diff --git a/ioreplay/src/init/itask.c b/ioreplay/src/init/itask.c new file mode 100644 index 0000000..f04ce33 --- /dev/null +++ b/ioreplay/src/init/itask.c @@ -0,0 +1,66 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "itask.h" + +itask_s* itask_new() +{ + itask_s *task = Malloc(itask_s); + + task->path = NULL; + itask_reset_stats(task); + + return task; +} + +void itask_destroy(itask_s *task) +{ + if (task->path) + free(task->path); + + free(task); +} + +void itask_reset_stats(itask_s *task) +{ + task->is_dir = task->is_file = false; + task->sizes_created = task->vsize = 0; + task->dirs_created = task->files_created = 0; + + if (task->path) { + free(task->path); + task->path = NULL; + } +} + +void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created, + long *files_total_size) +{ + *dirs_created += task->dirs_created; + *files_created += task->files_created; + *files_total_size += task->sizes_created; + + if (*dirs_created < 0 || *files_created < 0 || *files_total_size < 0) { + Error("Size overflow"); + } + + itask_reset_stats(task); +} + +void itask_print(itask_s *task) +{ + Put("itask(%p): is_dir:%d is_file:%d vsize:%ld path:%s", + (void*)task, task->is_dir, task->is_file, + task->vsize, task->path); +} diff --git a/ioreplay/src/init/itask.h b/ioreplay/src/init/itask.h new file mode 100644 index 0000000..b10d515 --- /dev/null +++ b/ioreplay/src/init/itask.h @@ -0,0 +1,72 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ITASK_H +#define ITASK_H + +#include "../defaults.h" + +/** + * @brief The initialise task definition + */ +typedef struct itask_s_ { + bool is_dir; + bool is_file; + long vsize; + char *path; + long dirs_created; + long files_created; + long sizes_created; +} itask_s; + +/** + * @brief Creates a new task object + * + * @return The new task object + */ +itask_s* itask_new(); + +/** + * @brief Resets the task stats + * + * @param task The itask object + */ +void itask_reset_stats(itask_s *task); + +/** + * @brief Extract stats from a task object + * + * @param task The itask object + * @param dirs_created Adds count of dirs created to that variable + * @param files_created Adds count of files created to that variable + * @param files_total_size Adds size of files created to that variable + */ +void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created, + long *files_total_size); + +/** + * @brief Destroys a given task object + * + * @param task The task object + */ +void itask_destroy(itask_s *task); + +/** + * @brief Prints a task to stdout + * + * @param task The task object + */ +void itask_print(itask_s *task); + +#endif // ITASK_H diff --git a/ioreplay/src/init/ithread.c b/ioreplay/src/init/ithread.c new file mode 100644 index 0000000..a580e70 --- /dev/null +++ b/ioreplay/src/init/ithread.c @@ -0,0 +1,99 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ithread.h" + +#include "itask.h" +#include "../utils/futils.h" + + +void* ithread_pthread_start(void *data) +{ + ithread_s *t = data; + init_s *i = t->init; + itask_s *task = NULL; + + do { + while (NULL != (task = rbuffer_get_next(t->queue))) { + ithread_run_task(t, task); + + // We need to mutex lock the reuse_queue as multiple threads + // can insert into it + pthread_mutex_lock(&i->reuse_queue_mutex); + int ret = rbuffer_insert(i->reuse_queue, task); + pthread_mutex_unlock(&i->reuse_queue_mutex); + if (!ret) + itask_destroy(task); + } + usleep(100); + } while (!t->terminate); + + while (NULL != (task = rbuffer_get_next(t->queue))) { + ithread_run_task(t, task); + if (!rbuffer_insert(i->reuse_queue, task)) + itask_destroy(task); + + pthread_mutex_lock(&i->reuse_queue_mutex); + int ret = rbuffer_insert(i->reuse_queue, task); + pthread_mutex_unlock(&i->reuse_queue_mutex); + if (!ret) + itask_destroy(task); + } + + return NULL; +} + +ithread_s* ithread_new(init_s *i) +{ + ithread_s *t = Malloc(ithread_s); + + t->init = i; + t->queue = rbuffer_new(1024); + t->terminate = false; + + return t; +} + +void ithread_start(ithread_s *t) +{ + start_pthread(&t->pthread, ithread_pthread_start, (void*)t); +} + +void ithread_destroy(ithread_s *t) +{ + rbuffer_destroy(t->queue); + free(t); +} + +void ithread_terminate(ithread_s *t) +{ + t->terminate = true; + pthread_join(t->pthread, NULL); +} + +void ithread_run_task(ithread_s *t, itask_s *task) +{ + if (task->is_dir) { + task->dirs_created += ensure_dir_exists(task->path); + + } else if (task->is_file) { + if (!ensure_file_exists(task->path, &task->dirs_created)) { + task->files_created++; + if (task->vsize > 0) { + append_random_to_file(task->path, task->vsize); + task->sizes_created += task->vsize; + } + } + } +} diff --git a/ioreplay/src/init/ithread.h b/ioreplay/src/init/ithread.h new file mode 100644 index 0000000..0884519 --- /dev/null +++ b/ioreplay/src/init/ithread.h @@ -0,0 +1,86 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ITHREAD_H +#define ITHREAD_H + +#include "../defaults.h" +#include "../datas/rbuffer.h" + +#include "init.h" +#include "itask.h" + +#include <pthread.h> + +/** + * @brief Definition of an init thread + * + */ +typedef struct ithread_s_ { + pthread_t pthread; /**< We run the init tasks in concurrent pthreads */ + rbuffer_s *queue; /**< The thread's task queue */ + init_s *init; /**< The responsible init object */ + bool terminate; /**< Indicates that thread can terminate */ +} ithread_s; + +/** + * @brief Creates a new thread object + * + * @param i The init object + * @return The new thread object + */ +ithread_s* ithread_new(init_s *i); + +/** + * @brief Terminates the thread + * + * This function waits (via join) for the pthread to complete all its + * current tasks from the queue. + * + * @param t The thread object + */ +void ithread_terminate(ithread_s* t); + +/** + * @brief Destroys the thread object + * + * @param t The thread object + */ +void ithread_destroy(ithread_s* t); + +/** + * @brief Executes the init task + * + * @param t The thread object + * @param task The task object + */ +void ithread_run_task(ithread_s* t, itask_s *task); + +/** + * @brief Starts the POSIX thread + * + * @param t The responsible thread object + */ +void ithread_start(ithread_s *t); + +/** + * @brief Entry point of the POSIX thread + * + * @param data Data passed to the pthread + * @return Always NULL on success + */ + +void* ithread_pthread_start(void *data); + +#endif // ITHREAD_H diff --git a/ioreplay/src/macros.h b/ioreplay/src/macros.h new file mode 100644 index 0000000..45e5a10 --- /dev/null +++ b/ioreplay/src/macros.h @@ -0,0 +1,116 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MACROS_H +#define MACROS_H + +#define Cleanup(code) ret = code; goto cleanup +#define Cleanup_unless(expr, code) \ + if (expr != code) { ret = code; goto cleanup; } + +// String helpers +#define Clone(str) notnull(strdup(str),__FILE__,__LINE__,0) +#define Eq(str1,str2) strcmp(str1,str2) == 0 + +// Number helpers +#define Abs(num) num >= 0 ? num : -num +#define Readhex(str) strtol(str, NULL, 16) +#define Perc(a, b) a > b ? b/(a/100.) : a/(b/100.) + +// Bitwise helpers +#define Has(flags, what) (flags & (what)) == (what) +#define Hasnt(flags, what) (flags & (what)) != (what) + +// Memory helpers +#define Malloc(what) \ + notnull(malloc(sizeof(what)),__FILE__,__LINE__,1) +#define Calloc(count,what) \ + notnull(calloc(count,sizeof(what)),__FILE__,__LINE__,count) +#define Mset(where,value,count,what) \ + memset(where,value,count*sizeof(what)) + +// Open helpers +#define Fopen(path, mode) fnotnull(fopen(path, mode), path, __FILE__, __LINE__) + +// Mmap helpers +#define Mmapshared(what) \ + mmapok(mmap(NULL, sizeof(what), \ + PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0), \ + __FILE__,__LINE__) +#define Cmapshared(count,what) \ + mmapok(mmap(NULL, count*sizeof(what), \ + PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0), \ + __FILE__,__LINE__) + +// Printing messages +#define Out(...) \ + fprintf(stdout, __VA_ARGS__); \ + fflush(stdout); +#define Put(...) \ + fprintf(stdout, __VA_ARGS__); \ + fprintf(stdout, "\n"); \ + fflush(stdout); + +// Printing debug messages +#define Debug(...) \ + fprintf(stderr, "%s:%d DEBUG: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + fflush(stderr); + +// Printing error messages +#define Error(...) \ + fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define Error_if(expr, ...) if (expr) { Error(__VA_ARGS__); } + +#define Errno(...) \ + fprintf(stderr, "%s:%d ERROR: %s (%d). ", __FILE__, __LINE__, \ + strerror(errno), errno); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define Errno_if(expr, ...) if (expr) { Errno(__VA_ARGS__); } + +#define Segfault(...) \ + fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); \ + *(int*)0 = 0; + +// Printing warn messages +#define Warn(...) \ + fprintf(stderr, "WARN: "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); + +#define Warn_if(expr, ...) if (expr) { Warn(__VA_ARGS__); } + +// Other helpers +#define Fill_with_stuff(buf, len) \ + for (int i = 0; i<len-1; ++i) { buf[i] = 'X'; } + +#endif // MACROS_H diff --git a/ioreplay/src/main.c b/ioreplay/src/main.c new file mode 100644 index 0000000..4a65de3 --- /dev/null +++ b/ioreplay/src/main.c @@ -0,0 +1,275 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file main.c + * @author Paul Buetow + * + * @brief The entry point of the I/O Replay program. + */ + +#include <signal.h> +#include <fcntl.h> + +#include "capture/capture.h" +#include "cleanup/cleanup.h" +#include "generate/generate.h" +#include "init/init.h" +#include "mounts.h" +#include "options.h" +#include "replay/replay.h" +#include "utests.h" +#include "utils/utils.h" + +/** + * @brief Do some architecture checks + * + * To ensure that I/O replay works correctly we have to check whether some + * data types are atomic or not. This is what this function does! + */ +static void _arch_check_atomic(void) +{ + if (sizeof(int) > sizeof(sig_atomic_t)) { + Error("int data type is not atomic on this architecture: %ld > %ld", + sizeof(int), sizeof(sig_atomic_t)); + + } else if (sizeof(bool) > sizeof(sig_atomic_t)) { + Error("bool data type is not atomic on this architecture: %ld > %ld", + sizeof(bool), sizeof(sig_atomic_t)); + } +} + +/** + * @brief Prints out version and copyright information + */ +static void _print_version(void) +{ + Put("This is I/O Replay %s - %s", IOREPLAY_VERSION, IOREPLAY_COPYRIGHT); +} + +/** + * @brief Print the synopsis + */ +static void _print_synopsis(void) +{ + _print_version(); + + Put("Synopsis:"); + Put("\tioreplay -c io.capture [-x PID] [-m MODULE]"); + Put("\tioreplay -c io.capture -r io.replay [-n str] [-u str] [-w str]"); + Put("\tioreplay -i io.replay"); + Put("\tioreplay -r io.replay [-p #] [-t #] [-D] [-s #]"); + Put("\tioreplay -R io.replay [-p #] [-t #] [-D] [-s #]"); + Put("\tioreplay -d"); + Put("\tioreplay -P"); + Put("\tioreplay -T [-n NAME]"); + Put("\tioreplay -V"); +} + +/** + * @brief Print a brief help + */ +static void _print_help(void) +{ + _print_synopsis(); + + Put("Help:"); + Put("\t-d Drop all Linux/FS caches and exit ioreplay"); + Put("\t-D Don't drop all caches (in conjunction with -r/-R):"); + Put("\t-s SPEED The speed factor (default: 0 [as fast as possible])"); + Put("\t-h Print this help"); + Put("\t-c FILE The capture file"); + Put("\t-n NAME The name (default: test0)"); + Put("\t-u USER The test run user (default: mcuser)"); + Put("\t-p #WORKERS Amount of of parallel worker processes (default: 4)"); + Put("\t-t #THREADS Threads per worker process (default: 128)"); + Put("\t-i REPLAYFILE The replay file to be initialised"); + Put("\t-r REPLAYFILE The replay file to be replayed"); + Put("\t-R REPLAYFILE Init and replay in one run (-i and -r combined)"); + Put("\t-S STATSFILE Write a stats file at the end of a test"); + Put("\t-T Trash data directories"); + Put("\t-P Purge all trash directories of all tests)"); + Put("\t-V Print I/O replay program version"); + Put("\t-w WD_BASE The working directory's base path"); + Put("\t (default: /usr/local/ioreplay)"); + Put("\t-x PID To specify a process ID (in conjunction with -c)"); + Put("\t-m MODULE To specify a module (in conjunction with -c)"); + Put("\nExample (run these commands one after another):"); + Put("\t 1.) sudo ioreplay -c io.capture"); + Put("\t 2.) sudo ioreplay -r io.replay -c io.capture -u paul -n test1"); + Put("\t 3.) sudo ioreplay -i io.replay"); + Put("\t 4.) sudo ioreplay -r io.replay -S"); +} + +/** + * @brief I/O Replay's entry point + * + * Not much more to document here though! + * @return The exit code + */ +int main(int argc, char **argv) +{ + _arch_check_atomic(); + status_e ret = UNKNOWN; + + bool dont_drop_caches = false; + options_s *opts = options_new(); + int opt = 0; + + while ((opt = getopt(argc, argv, "Vr:R:S:c:u:i:hw:n:dDs:w:p:t:UPTx:m:")) != -1) { + switch (opt) { + case 'U': + utests_run(); + Cleanup(SUCCESS); + break; + case 'V': + _print_version(); + Cleanup(SUCCESS); + break; + case 'd': + drop_caches(); + Cleanup(SUCCESS); + break; + case 'D': + dont_drop_caches = true; + break; + case 'c': + opts->capture_file = absolute_path(optarg); + Put("Capture file: %s", opts->capture_file); + break; + case 'P': + opts->purge = true; + Put("Purge option set"); + break; + case 'T': + opts->trash = true; + Put("Trash option set"); + break; + case 'i': + opts->init = true; + if (!opts->replay_file) { + opts->replay_file = absolute_path(optarg); + Put("Replay file: %s", opts->replay_file); + } + break; + case 'R': + opts->init = true; + opts->replay = true; + if (!opts->replay_file) { + opts->replay_file = absolute_path(optarg); + Put("Replay file: %s", opts->replay_file); + } + break; + case 'r': + opts->replay = true; + if (!opts->replay_file) { + opts->replay_file = absolute_path(optarg); + Put("Replay file: %s", opts->replay_file); + } + break; + case 'S': + opts->stats_file = Clone(optarg); + Put("Stats output file: %s", opts->stats_file); + break; + case 'w': + opts->wd_base = optarg; + Put("WD base: %s", opts->wd_base); + break; + case 'u': + opts->user = optarg; + Put("User: %s", opts->user); + break; + case 'm': + opts->module = Clone(optarg); + Put("Module: %s", opts->module); + break; + case 'n': + opts->name = optarg; + Put("Name: %s", opts->name); + break; + case 'h': + _print_help(); + Cleanup(SUCCESS); + case 's': + sscanf(optarg, "%lf", &opts->speed_factor); + Put("Speed factor: %lf", opts->speed_factor); + break; + case 'p': + opts->num_workers = atoi(optarg); + if (opts->num_workers < 1) + opts->num_workers = 1; + Put("Num worker processes: %d", opts->num_workers); + break; + case 't': + opts->num_threads_per_worker = atoi(optarg); + if (opts->num_threads_per_worker < 1) + opts->num_threads_per_worker = 1; + Put("Num threads per worker: %d", opts->num_threads_per_worker); + break; + case 'x': + opts->pid = atoi(optarg); + Put("PID: %d", opts->pid); + break; + default: + _print_help(); + Cleanup(ERROR); + } + } + + if (opts->purge || opts->trash) { + // Clean up all temp data of previous test runs + Cleanup(cleanup_run(opts)); + + } else if (opts->capture_file && !opts->replay_file) { + // We are going to capture I/O + Cleanup(capture_run(opts)); + + } else if (opts->capture_file && opts->replay_file) { + // We are going to generate a .replay file from the .capture file + Cleanup(generate_run(opts)); + + } else if (opts->replay_file && opts->init && !opts->replay) { + // We are going to initialise the test from the .replay file! + Cleanup(init_run(opts)); + + } else if (opts->replay_file && opts->init && opts->replay) { + // We are going to initialise the test and run the test! Run the + // initialiser in a sub-process, as it drops root privileges! + pid_t pid = fork(); + if (pid == 0) { + Cleanup(init_run(opts)); + } else { + opts->drop_caches = !dont_drop_caches; + int init_status; + waitpid(pid, &init_status, 0); + // Only proceed if initialisation was successfull! + Cleanup_unless(SUCCESS, init_status); + Cleanup(replay_run(opts)); + } + + } else if (opts->replay_file && !opts->init && opts->replay) { + // We are going to replay the I/O + opts->drop_caches = !dont_drop_caches; + Cleanup(replay_run(opts)); + + } else { + _print_help(); + Cleanup(ERROR); + } + +cleanup: + options_destroy(opts); + + return ret; +} diff --git a/ioreplay/src/meta/meta.c b/ioreplay/src/meta/meta.c new file mode 100644 index 0000000..d56c17e --- /dev/null +++ b/ioreplay/src/meta/meta.c @@ -0,0 +1,111 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "meta.h" + +#define _MAX_META_LEN 256 + +meta_s* meta_new(FILE *replay_fd) +{ + meta_s *m = Malloc(meta_s); + + m->replay_fd = replay_fd; + m->offset = ftello(replay_fd); + m->read_buf = NULL; + + return m; +} + +void meta_destroy(meta_s *m) +{ + if (!m) + return; + + if (m->read_buf) + free(m->read_buf); + + free(m); +} + +void meta_reserve(meta_s *m) +{ + // TODO: Use a hole in the .replay file to reserve space + char buf[_MAX_META_LEN]; + Mset(&buf, '#', _MAX_META_LEN-1, char); + fprintf(m->replay_fd, "%s\n", buf); +} + +void meta_write_start(meta_s *m) +{ + fseeko(m->replay_fd, m->offset, SEEK_SET); + // Write required '#' so that the regular worker processes + // will ignore that meta line. + fprintf(m->replay_fd, "#"); + + // Required for parsing in 'meta_read_s' + fprintf(m->replay_fd, "|"); +} + +void meta_write_s(meta_s *m, char *key, char *val) +{ + fprintf(m->replay_fd, "%s=%s|", key, val); +} + +void meta_write_l(meta_s *m, char *key, long val) +{ + char buf[1024]; + sprintf(buf, "%ld", val); + fprintf(m->replay_fd, "%s=%ld|", key, val); +} + +void meta_read_start(meta_s *m) +{ + size_t len = 0; + m->read_buf = Calloc(_MAX_META_LEN, char); + getline(&m->read_buf, &len, m->replay_fd); +} + +bool meta_read_s(meta_s *m, char *key, char **val) +{ + char *saveptr = NULL; + char *iterate_buf = Clone(m->read_buf); + int keylen = strlen(key); + + char *tok = strtok_r(iterate_buf, "|", &saveptr); + + while (tok) { + if (strncmp(tok, key, keylen) == 0 && tok[keylen] == '=') { + asprintf(val, "%s", tok+keylen+1); + free(iterate_buf); + return true; + } + tok = strtok_r(NULL, "|", &saveptr); + } + + free(iterate_buf); + return false; +} + +bool meta_read_l(meta_s *m, char *key, long *val) +{ + char *buf = NULL; + + if (meta_read_s(m, key, &buf)) { + *val = atol(buf); + free(buf); + return true; + } + + return false; +} diff --git a/ioreplay/src/meta/meta.h b/ioreplay/src/meta/meta.h new file mode 100644 index 0000000..10002cc --- /dev/null +++ b/ioreplay/src/meta/meta.h @@ -0,0 +1,107 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef META_H +#define META_H + +#include "../defaults.h" + +/** + * @brief The meta information definition + * + * This is used to write or read meta information to/from the header + * of the .replay file. This information then is used by ioreplay + * in other steps. E.g. reading the amount of used file descriptors + * from the meta header in order to allocate data structures of the + * correct sizes before running the test! + */ +typedef struct meta_s_ { + FILE* replay_fd; /**< The FS of the .replay file */ + off_t offset; /**< The meta offset (usually 0) */ + char* read_buf; /**< Pointer to a read buffer */ +} meta_s; + +/** + * @brief Creates a new meta bject + * + * @return The new meta object + */ +meta_s* meta_new(); + +/** + * @brief Destroys a meta object + * + * @param m The meta object + */ +void meta_destroy(meta_s *m); + +/** + * @brief Reserves space in the .replay file for the meta header + * + * @param m The meta object + */ +void meta_reserve(meta_s *m); + +/** + * @brief Indicates that we start writing the meta header to the .replay file + * + * @param m The meta object + */ +void meta_write_start(meta_s *m); + +/** + * @brief Writes a string to the meta header + * + * @param m The meta object + * @param key The key + * @param val The string value + */ +void meta_write_s(meta_s *m, char *key, char *val); + +/** + * @brief Writes a long to the meta header + * + * @param m The meta object + * @param key The key + * @param val The long value + */ +void meta_write_l(meta_s *m, char *key, long val); + +/** + * @brief indicates that we start reading from the meta header + * + * @param m The meta object + */ +void meta_read_start(meta_s *m); + +/** + * @brief Reads a string from the meta header + * + * @param m The meta object + * @param key The key + * @param val The string val read + */ +bool meta_read_s(meta_s *m, char *key, char **val); + +/** + * @brief Reads a long from the meta header + * + * @param m The meta object + * @param key The key + * @param val The long val read + */ +bool meta_read_l(meta_s *m, char *key, long *val); + +#endif // META_H + diff --git a/ioreplay/src/mounts.c b/ioreplay/src/mounts.c new file mode 100644 index 0000000..ac6f1d4 --- /dev/null +++ b/ioreplay/src/mounts.c @@ -0,0 +1,400 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mounts.h" + +#include "utils/futils.h" + +#define _PATH_INSERT "/.ioreplay/" +#define _PATH_INSERT_LEN 11 // strlen of _PATH_INSERT + +void mounts_read(mounts_s *m) +{ + char *mounts = "/proc/mounts"; + size_t len = 0; + char *line = NULL; + char *saveptr = NULL; + + Put("Reading '%s'", mounts); + + FILE *fp = Fopen(mounts, "r"); + Out("Adding supported file systems to replay paths:"); + + while (getline(&line, &len, fp) != -1) { + bool ignore = true; + + char *dev = strtok_r(line, " ", &saveptr); + if (dev == NULL) { + Error("Could not parse device from %s", mounts); + } + + char *mp = strtok_r(NULL, " ", &saveptr); + if (mp == NULL) { + Error("Could not parse mountpoint from %s", mounts); + } + + char *fs = strtok_r(NULL, " ", &saveptr); + if (fs == NULL) { + Error("Could not parse file system from %s", mounts); + } +#ifdef MP_DEBUG + Debug("fs:%s", fs); +#endif + // TODO: Make file system types configurable + if (Eq(fs, "ext2")) { + ignore = false; + } else if (Eq(fs, "ext5")) { + ignore = false; + } else if (Eq(fs, "ext4")) { + ignore = false; + } else if (Eq(fs, "xfs")) { + ignore = false; + } else if (Eq(fs, "zfs")) { + ignore = false; + } else if (Eq(fs, "btrfs")) { + ignore = false; + } + + if (ignore) { + if (strcmp(mp, "/") != 0) { + m->ignore_mps[m->ignore_count] = Clone(mp); + m->ignore_count++; + } + + } else if (m->count >= MAX_MOUNTPOINTS) { + Error("Exceeded max mount points: %d\n", m->count); + + } else { + Out(" %s (%s)", mp, fs); + m->mps[m->count] = Clone(mp); + m->lengths[m->count] = strlen(mp); + m->count++; + } + } + + fclose(fp); + Out("\n"); +} + +mounts_s *mounts_new(options_s *opts) +{ + mounts_s *m = Malloc(mounts_s); + + m->opts = opts; + m->count = 0; + m->ignore_count = 0; + mounts_read(m); + + return m; +} + +void mounts_destroy(mounts_s *m) +{ + if (!m) + return; + for (int i = 0; i < m->count; i++) + free(m->mps[i]); + free(m); +} + +void mounts_trash(mounts_s *m) +{ + options_s *opts = m->opts; + drop_root(opts->user); + Put("Moving all old files to trash (of previous tests)..."); + + struct timeval tv; + gettimeofday(&tv, NULL); + + char *wd_path = NULL; + asprintf(&wd_path, "%s/%s", opts->wd_base, opts->name); + + char *trash_path = NULL; + asprintf(&trash_path, "%s/.trash/%ld", opts->wd_base, tv.tv_sec); + + if (is_dir(wd_path)) { + ensure_dir_exists(trash_path); + chown_path(opts->user, trash_path); + if (rename(wd_path, trash_path)) { + Errno("Could not move '%s' to '%s'", wd_path, trash_path); + } + } + free(wd_path); + free(trash_path); + + for (int i = 0; i < m->count; i++) { + char *mp = m->mps[i]; + char *path = NULL; + asprintf(&path, "%s/%s/%s", mp, _PATH_INSERT, opts->name); + asprintf(&trash_path, "%s/%s/.trash/%ld", + mp, _PATH_INSERT, tv.tv_sec); + + if (is_dir(path)) { + ensure_dir_exists(trash_path); + chown_path(opts->user, trash_path); + if (rename(path, trash_path)) { + Errno("Could not move '%s' to '%s'", path, trash_path); + } + } + + free(path); + free(trash_path); + } + + Put("Done trashing!"); + Put("Once the drives fill up you may want to purge old data (-P)"); +} + +void mounts_purge(mounts_s *m) +{ + options_s *opts = m->opts; + drop_root(opts->user); + + Out("Purging all data from the following directories:"); + + int active_purgers = 0, max_purgers = 16; + if (opts->num_workers > max_purgers) + max_purgers = opts->num_workers; + + char *purge_path = NULL; + asprintf(&purge_path, "%s", opts->wd_base); + if (is_dir(purge_path)) { + Out(" %s", purge_path); + pid_t pid = fork(); + + if (pid == 0) { + ensure_dir_empty(purge_path); + free(purge_path); + exit(0); + + } else if (pid < 0) { + Errno("\nUnable to create cleaner process! :'-("); + } + active_purgers++; + } + free(purge_path); + + int cleaner_status = SUCCESS; + + for (int i = 0; i < m->count; i++) { + char *mp = m->mps[i]; + char *purge_path = NULL; + asprintf(&purge_path, "%s/%s", mp, _PATH_INSERT); + + if (is_dir(purge_path)) { + if (active_purgers+1 >= max_purgers) { + wait(&cleaner_status); + active_purgers--; + } + + // TODO: Use threading model same way as in init/init.c + pid_t pid = fork(); + if (pid == 0) { + Out(" %s", purge_path); + ensure_dir_empty(purge_path); + free(purge_path); + exit(0); + } else if (pid < 0) { + Errno("Unable to create cleaner process! :'-("); + } + active_purgers++; + } + free(purge_path); + } + + while (wait(&cleaner_status) > 0) + active_purgers--; + Put("\nCleaning done!"); +} + +void mounts_init(mounts_s *m) +{ + options_s *opts = m->opts; + char *wd_path = NULL; + asprintf(&wd_path, "%s/%s", opts->wd_base, opts->name); + ensure_dir_exists(wd_path); + chown_path(opts->user, opts->wd_base); + chown_path(opts->user, wd_path); + + if (chdir(wd_path)) { + Errno("Could not chdir into '%s'!", wd_path); + + } else { + Put("Chdir into '%s'", wd_path); + } + + free(wd_path); + + for (int i = 0; i < m->count; i++) { + char *mp = m->mps[i]; + char *path = NULL; + + // Create .ioreplay/ directory on MP + asprintf(&path, "%s/%s", mp, _PATH_INSERT); + ensure_dir_exists(path); + chown_path(m->opts->user, path); + free(path); + path = NULL; + + // Create .ioreplay/NAME directory on MP + asprintf(&path, "%s/%s/%s", mp, _PATH_INSERT, opts->name); + ensure_dir_exists(path); + chown_path(m->opts->user, path); + free(path); + } +} + +bool mounts_ignore_path(mounts_s *m, const char *path) +{ + // CentOS 7 specific, ignore temp namespace mounts! + char *pos = strstr(path, "/tmp/namespace-"); + if (pos == path) + return true; + + // iterate backwards through all mount points. + for (int i = m->ignore_count-1; i >= 0; --i) { + char *mountpoint = m->ignore_mps[i]; + pos = strstr(path, mountpoint); + // Ignore this path as it is in the ignore mp list + if (pos == path) + return true; + } + + return false; +} + +bool mounts_transform_path(mounts_s *m, const char *name, + char *path, char **path_r) +{ + char *tmp = NULL; +#ifdef DEBUG_TRANSFORM_PATH + char *original_path = path; +#endif + bool line_ok = true; + + // First figure out whether there are '..' in any paths. If so we have to + // tokenize the path and remove '..'. Example: + // transform '/foo/bar/../' into '/foo/'. + // Also remove double '/' from paths. + + if (strstr(path, "..") || strstr(path, "//")) { + // tmp will be freed under label 'cleanup' at end of function. + tmp = Calloc(strlen(path)+1, char); + + // stack to put the tokens on + stack_s *s = stack_new(); + + // we need a copy of the path, so we can tokenize it into the stack + char* clone = Clone(path); + + char *saveptr = NULL; + char *tok = strtok_r(clone, "/", &saveptr); + + // Add each part of the path to the stack. + while (tok) { + if (strcmp(tok, "..") == 0) { + stack_pop(s); + } else { + stack_push(s, tok); + } + tok = strtok_r(NULL, "/", &saveptr); + } + + if (stack_is_empty(s)) { + strcpy(tmp, "."); + + } else { + s = stack_new_reverse_from(s); + strcpy(tmp, "/"); + strcat(tmp, (char*)stack_pop(s)); + + while(!stack_is_empty(s)) { + strcat(tmp, "/"); + strcat(tmp, (char*)stack_pop(s)); + } + } + + stack_destroy(s); + free(clone); + + // This is the path without '..' and '//' (and '///' ... etc') + path = tmp; + } + + // Now heck whether the path is on a supported file system. If not, ignore! + if (mounts_ignore_path(m, path)) { + line_ok = false; + goto cleanup; + } + + // So the path is on a valid mount point! Now we need to insert + // .ioreplay/NAME to each mount point, e.g. /usr/local/.ioreplay/NAME/... + + // Iterate backwards through all mount points. + for (int i = m->count-1; i >= 0; --i) { + char *mountpoint = m->mps[i]; + int mp_len = m->lengths[i]; + + if (strncmp(path, mountpoint, mp_len) == 0) { + // Found a path to replace + // Now insert .ioreplay/NAME/ into the file path. + *path_r = Calloc(strlen(path) + strlen(name)+1 + + _PATH_INSERT_LEN+1, char); + + if (strcmp(mountpoint, "/") == 0) { + // Root path + strcpy(*path_r, _PATH_INSERT); + strcat(*path_r, name); + strcat(*path_r, path); + + } else { + strcpy(*path_r, mountpoint); + strcat(*path_r, _PATH_INSERT); + strcat(*path_r, name); + char *pos = path; + pos += mp_len * (int) sizeof(char); + strcat(*path_r, pos); + } + + goto cleanup; + } + } + + if (tmp) + free(tmp); + + return line_ok; + +cleanup: +#ifdef DEBUG_TRANSFORM_PATH + Debug("Transform path '%s' -> '%s' -> '%s'", original_path, path, *path_r); +#endif + if (tmp) + free(tmp); + + return line_ok; +} + +int mounts_get_mountnumber(mounts_s *m, const char *path) +{ + for (int i = m->count-1; i >= 0; --i) { + char *mountpoint = m->mps[i]; + int mp_len = m->lengths[i]; + + if (strncmp(path, mountpoint, mp_len) == 0) + return i; + } + + return 0; +} diff --git a/ioreplay/src/mounts.h b/ioreplay/src/mounts.h new file mode 100644 index 0000000..a644ddb --- /dev/null +++ b/ioreplay/src/mounts.h @@ -0,0 +1,154 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MOUNTPOINTS_H +#define MOUNTPOINTS_H + +#include "datas/stack.h" +#include "defaults.h" +#include "options.h" + +#define MAX_MOUNTPOINTS 1024 + +/** + * @brief Represents data parsed from /proc/mounts + * + * This is used to determine the file systems and the file system types + * currently mounted on the Linux system. I/O replay only replays I/O + * on specific file systems such as ext4 or xfs and will ignore any special + * or pseudo file systems such as tmpfs, devfs, sysfs. It does not make sense + * to replay I/O on these because there is actually no underlying block device + * attached to these. + * + * A mounts object helps to determine whether a path relies on a valid file + * system or not. All I/O operations on invalid file systems are being filtered + * out! + * + * The mounts object also does more things such as purging temp test data from + * the mountpoints etc... + */ +typedef struct mounts_s_ { + int count; /**< The amount of mount points */ + char *mps[MAX_MOUNTPOINTS]; /**< The mp paths */ + int lengths[MAX_MOUNTPOINTS]; /**< The mp lenghts */ + int ignore_count; /**< The amount of ignored mount points */ + char *ignore_mps[MAX_MOUNTPOINTS]; /**< The ignored mp paths */ + options_s *opts; /**< A pointer to the options object */ +} mounts_s; + +/** + * @brief Creates a new mounts object + * + * @param opts The options object + * @return The new mounts object + */ +mounts_s *mounts_new(options_s *opts); + +/** + * @brief Destroys the mounts object + * + * @param m The mounts object + */ +void mounts_destroy(mounts_s *m); + +/** + * @brief moves all files within replay mounts to trash + * + * It moves all files of the .ioreplay/NAME directories to + * .ioreplay/NAME.trashEPOCH directories for all available mount points. + * It does the same for the working dorectory of the current test. + * + * @param m The responsible mounts object + */ +void mounts_trash(mounts_s *m); + +/** + * @brief Deletes all files within replay mounts + * + * It deletes all files from the .ioreplay/ directories for all availabe + * mount points. It also deletes the working directory of all tests. The + * function forks one sub-process per mount point, so it is cleaning all drives + * in parallel. + * + * It can take a significant amount of time to actually delete all these files. + * That's why there is also a mounts_trash function, which will not delete the + * files but move them to trash folders so they can be deleted at a later + * point. + * + * @param m The responsible mounts object + */ +void mounts_purge(mounts_s *m); + +/** + * @brief Ensures all mounts have a .ioreplay/NAME directory + * + * These directories are used by ioreplay to run the I/O replay tests in. + * The function also ensures to have the correct user permissions for these + * directories. + * + * @param m The responsible mounts object + */ +void mounts_init(mounts_s *m); + +/** + * @brief Reads /proc/mounts to determine which mounts are available + * + * @param m The mounts object + */ +void mounts_read(mounts_s *m); + +/** + * @brief Determines whether a path should be ignored + * + * ioreplay replays I/O only on known mount points of known + * file system types. This function helps to determine whether + * a path is on a valid mount point or not. + * + * @param m The responsible mounts object + * @param path The path to check + * @return true if path has to be ignored + */ +bool mounts_ignore_path(mounts_s *m, const char *path); + +/** + * @brief Inserts ./ioreplay/NAME into a path + * + * This function inserts ./ioreplay/NAME into a given file path. + * The function also checks whether the path is on a supported replay + * path or not. E.g. we want to ignore file systems such as devfs, sysfs, + * procfs.. etc. + * + * @param m The responsible mountpoint object + * @param name The name of the test + * @param path The original path + * @param path_r The tansformed path (has to be freed if not NULL) + * @return False if this path is to be ignored + */ +bool mounts_transform_path(mounts_s *m, const char *name, + char *path, char **path_r); + + +/** + * @brief Get's the mount point number of a path + * + * Used by init.c to determine which thread to use to initialise a file + * or directory on a given path. + * + * @param m The responsible mountpoint object + * @param path The file/directory path + * @return The mountpoint number + */ +int mounts_get_mountnumber(mounts_s *m, const char *path); + +#endif // MOUNTPOINTS_H diff --git a/ioreplay/src/opcodes.h b/ioreplay/src/opcodes.h new file mode 100644 index 0000000..3d5c114 --- /dev/null +++ b/ioreplay/src/opcodes.h @@ -0,0 +1,103 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPCODES_H +#define OPCODES_H + +typedef enum { + // stat() syscalls + FSTAT = 0, + FSTAT_AT, + FSTATFS, + FSTATFS64, + LSTAT, + STAT, + STATFS, + STATFS64, + + // read() syscalls + READ = 10, + READV, + READAHEAD, + READDIR, + READLINK, + READLINK_AT, + + // write() syscalls + WRITE = 20, + WRITEV, + + // open() and other syscalls which may create files + OPEN = 30, + OPEN_AT, + CREAT, + MKDIR, + MKDIR_AT, + NAME_TO_HANDLE_AT, + OPEN_BY_HANDLE_AT, + + // rename() syscalls + RENAME = 40, + RENAME_AT, + RENAME_AT2, + + // close() and unlink() syscalls + CLOSE = 50, + UNLINK, + UNLINK_AT, + RMDIR, + + // sync() syscalls + FSYNC = 60, + FDATASYNC, + SYNC, + SYNCFS, + SYNC_FILE_RANGE, + + // other syscalls + FCNTL = 70, + GETDENTS, + LSEEK, + + // mmap syscalls + MMAP2 = 80, + MUNMAP, + REMAP, + MSYNC, + + // chmod() syscalls + CHMOD = 100, + FCHMOD, + FCHMODAT, + + // chown() syscalls + CHOWN = 110, + CHOWN16, + LCHOWN, + LCOWN16, + FCHOWN, + FCHOWN16, + FCHOWNAT, + + // Meta operations (I/O replay internal use only) + // A single thread terminates + META_EXIT = 900, + // All threads of a process termiate (process termination) + META_EXIT_GROUP, + // Meta operation for lamport synchronisation (currently unused) + META_TIMELINE + +} opcode_e; + +#endif // OPCODES_H diff --git a/ioreplay/src/options.c b/ioreplay/src/options.c new file mode 100644 index 0000000..c1dcdb9 --- /dev/null +++ b/ioreplay/src/options.c @@ -0,0 +1,51 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "options.h" + +options_s *options_new() +{ + options_s *o = Malloc(options_s); + + o->capture_file = NULL; + o->replay_file = NULL; + o->stats_file = NULL; + o->wd_base = "/usr/local/ioreplay"; + o->num_workers = 4; + o->num_threads_per_worker = 128; + o->user = "mcuser"; + o->name = "test0"; + o->init = false; + o->replay = false; + o->speed_factor = 0; + o->drop_caches = false; + o->purge = false; + o->trash = false; + o->pid = -1; + o->module = "ioreplay.ko"; + + return o; +} + +void options_destroy(options_s *o) +{ + if (o->capture_file) + free(o->capture_file); + if (o->replay_file) + free(o->replay_file); + if (o->stats_file) + free(o->stats_file); + + free(o); +} diff --git a/ioreplay/src/options.h b/ioreplay/src/options.h new file mode 100644 index 0000000..66cb0f7 --- /dev/null +++ b/ioreplay/src/options.h @@ -0,0 +1,61 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPTIONS_H +#define OPTIONS_H + +#include <stdbool.h> +#include "defaults.h" + +/** + * @brief The options definition, used to store user input + */ +typedef struct options_s_ { + char *capture_file; /**< The name of the .capture file */ + char *replay_file; /**< The name of the .replay file */ + char *stats_file; /**< The name of the .stats file */ + bool write_stats_file; /**< Write a stats file at the end of the test */ + char *user; /**< The user name to run the test as */ + char *name; /**< The name of the test (found in .ioreplay/name sub-dirs) */ + char *wd_base; /**< The working directory base */ + int num_workers; /**< The amount of worker processes */ + int num_threads_per_worker; /**< Max threads per worker processes */ + bool init; /**< If set ioreplay will initialise the environment */ + bool replay; /**< If set ioreplay will run/replay the test */ + bool purge; /**< If set ioreplay will purge the environment */ + bool trash; /**< If set ioreplay will trash the environment */ + bool drop_caches; /**< True if ioreplay should drop all Linux caches */ + double speed_factor; /**< Specifies how fast the test is replayed */ + int pid; /**< Specifies a process id to capture */ + char *module; /**< Specifies the kernel module for capturing */ +} options_s; + +/** + * @brief Creates a new options object + * + * The options object contains all options specified by the user as a command + * line option. It is filled with default values during creation. + * + * @return The options object + */ +options_s *options_new(); + +/** + * @brief Destroys the options object + * + * @param o The options object + */ +void options_destroy(options_s *o); + +#endif // OPTIONS_H diff --git a/ioreplay/src/replay/replay.c b/ioreplay/src/replay/replay.c new file mode 100644 index 0000000..89f5fee --- /dev/null +++ b/ioreplay/src/replay/replay.c @@ -0,0 +1,191 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "replay.h" + +#include "../datas/amap.h" +#include "../meta/meta.h" +#include "../mounts.h" +#include "rworker.h" +#include "rstats.h" + +void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes, + long *num_pids, long *num_fds, long *num_lines) +{ + meta_s *m = meta_new(replay_fd); + meta_read_start(m); + + long version = 0; + if (meta_read_l(m, "version", &version)) { + Put("Replay version is '%ld'", version); + if (version != REPLAY_VERSION) { + Error(".replay file of incompatible version, got %x, expected %x", + (int)version, REPLAY_VERSION); + } + } + + char *user; + if (meta_read_s(m, "user", &user)) { + Put("Setting user to '%s'", user); + opts->user = user; + } + + char *name; + if (meta_read_s(m, "name", &name)) { + Put("Setting name to '%s'", name); + opts->name = name; + } + + if (meta_read_l(m, "num_vsizes", num_vsizes)) { + if (*num_vsizes < 0) { + Error("Lamport vsize overflow"); + } + Put("Setting num of vsizes to '%ld'", *num_vsizes); + } + + if (meta_read_l(m, "num_mapped_pids", num_pids)) { + if (*num_pids < 0) { + Error("Process overflow (too many process IDs in .replay)"); + } + Put("Setting num of PIDs to '%ld'", *num_pids); + } + + if (meta_read_l(m, "num_mapped_fds", num_fds)) { + if (*num_fds < 0) { + Error("FD overflow (too many FDs in .replay)"); + } + Put("Setting num of FDs to '%ld'", *num_fds); + } + + if (meta_read_l(m, "num_lines", num_lines)) { + if (*num_fds < 0) { + Error("Overflow (too many lines in .replay)"); + } + Put("Setting num of lines to '%ld'", *num_lines); + } + + meta_destroy(m); +} + +status_e replay_run(options_s *opts) +{ + status_e status = SUCCESS; + + if (opts->drop_caches) { + drop_caches(); + //cache_file(opts->replay_file); + } + + // Extract information from the meta header + FILE *replay_fd = Fopen(opts->replay_file, "r"); + long num_vsizes = 0, num_pids = 0, num_fds = 0, num_lines = 0; + replay_extract_header(opts, replay_fd, &num_vsizes, &num_pids, + &num_fds, &num_lines); + fclose(replay_fd); + + // A map of all file descriptors used. + Out("Creating FD map..."); + amap_s *fds_map = NULL; + if (opts->num_workers > 1) { + fds_map = amap_new_mmapped(num_fds); + } else { + fds_map = amap_new(num_fds); + } + Put("done"); + + // To collect all individual worker's stats into the global + // stats object. + stack_s *all_worker_stats = stack_new(); + + // The global stats object + rstats_s *stats = rstats_new(opts); + rstats_start(stats); + + // Fork worker processes, each worker process will read the .replay file + // individually. + + if (opts->num_workers > 1) { + for (int i = 0; i < opts->num_workers; ++i) { + rworker_stats_s *worker_stats = rworker_stats_new_mmap(); + stack_push(all_worker_stats, worker_stats); + + pid_t pid = fork(); + + if (pid == 0) { + // One worker object per fork + rworker_s *w = rworker_new(i, fds_map, num_vsizes, num_pids, opts, + worker_stats); + + // Process the .replay journal line by line + status_e status = rworker_process_lines(w, num_lines); + Put("worker(%d): Exiting from %d with status %d", i, + pid, status); + rworker_destroy(w); + + // Exit sub-process + exit(status); + + } else if (pid < 0) { + Errno("worker(%d): Unable to create worker process! :'-(", i); + + } else { + Put("worker(%d): Process with pid %d forked", i, pid); + } + } + + drop_root(opts->user); + + Put("Waiting for worker processes to finish"); + pid_t pid; + int rworker_status = SUCCESS; + + while ((pid = wait(&rworker_status)) > 0) { + if (rworker_status != SUCCESS) + status = rworker_status; + + Put("Process with pid %d exited with status %d", + pid, rworker_status); + } + + Put("All workers finished (%d)!", status); + + } else { + Put("Only one worker, don't fork sub-processes"); + + rworker_stats_s *worker_stats = rworker_stats_new_mmap(); + stack_push(all_worker_stats, worker_stats); + + rworker_s *w = rworker_new(0, fds_map, num_vsizes, num_pids, + opts, worker_stats); + status = rworker_process_lines(w, num_lines); + rworker_destroy(w); + + Put("Worker finished work!"); + } + + // Collect all statistics + rstats_stop(stats); + while (!stack_is_empty(all_worker_stats)) { + rworker_stats_s *worker_stats = stack_pop(all_worker_stats); + rstats_add_from_worker(stats, worker_stats); + rworker_stats_destroy(worker_stats); + } + stack_destroy(all_worker_stats); + + rstats_print(stats); + rstats_destroy(stats); + + amap_destroy(fds_map); + return status; +} diff --git a/ioreplay/src/replay/replay.h b/ioreplay/src/replay/replay.h new file mode 100644 index 0000000..dcc3d84 --- /dev/null +++ b/ioreplay/src/replay/replay.h @@ -0,0 +1,46 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef REPLAY_H +#define REPLAY_H + +#include "../defaults.h" +#include "../utils/futils.h" +#include "../opcodes.h" +#include "../options.h" +#include "rioop.h" +#include "rprocess.h" + +/** + * @brief Replays the given .replay file + * + * @param opts The options object + * @return SUCCESS if everything went fine + */ +status_e replay_run(options_s *opts); + +/** + * @brief Extract required meta data from .replay's meta header + * + * @param opts The options object + * @param replay_fd The file handle to the .replay file + * @param num_vsizes The amount of virtual sizes/paths + * @param num_pids The amount of process IDs + * @param num_fds The amount of virtual file descriptors + * @param num_lines The amount of .replay lines with I/O ops + */ +void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes, + long *num_pids, long *num_fds,long *num_lines); + +#endif // REPLAY_H diff --git a/ioreplay/src/replay/rioop.c b/ioreplay/src/replay/rioop.c new file mode 100644 index 0000000..2e16c94 --- /dev/null +++ b/ioreplay/src/replay/rioop.c @@ -0,0 +1,425 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rioop.h" + +#include "../vfd.h" +#include "rworker.h" + +// Printing error messages +#define _Error(...) \ + fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\nlineno:%ld path:%s\n", task->lineno, vfd->path); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define _Errno(...) \ + fprintf(stderr, "%s:%d ERROR: %s (%d). ", __FILE__, __LINE__, \ + strerror(errno), errno); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\nlineno:%ld path:%s\n", task->lineno, vfd->path); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define _Init_arg(num) int arg = atoi(task->toks[num]) +#define _Init_cmd(num) int cmd = atoi(task->toks[num]) +#define _Init_fd(num) long fd = atol(task->toks[num]) +#define _Init_flags(num) int flags = atoi(task->toks[num]) +//#define _Init_mode(num) int mode = atoi(task->toks[num]) +#define _Init_offset(num) long offset = atol(task->toks[num]) +#define _Init_op(num) int op = atoi(task->toks[num]) +#define _Init_path2(num) char *path2 = task->toks[num] +#define _Init_path(num) char *path = task->toks[num] +#define _Init_rc(num) int rc = atoi(task->toks[num]) +#define _Init_whence(num) long whence = atol(task->toks[num]) + +#define _Init_bytes(num) \ + int bytes = atoi(task->toks[num]); \ + if (bytes <= 0) return + +#define _Init_virtfd \ + vfd_s *vfd = amap_get(p->fds_map, fd); \ + if (vfd == NULL) return + +void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_op(2); + + switch (op) { + // stat() syscalls + case FSTAT: + rioop_fstat(p, t, task); + break; + case FSTATFS: + case FSTATFS64: + //Error("op(%d) not implemented", op); + break; + case FSTAT_AT: + case LSTAT: + case STAT: + rioop_stat(p, t, task); + break; + case STATFS: + case STATFS64: + //Error("op(%d) not implemented", op); + break; + + // read() syscalls + case READ: + case READV: + rioop_read(p, t, task); + break; + case READAHEAD: + //Error("op(%d) not implemented", op); + break; + case READLINK: + case READLINK_AT: + //Error("op(%d) not implemented", op); + break; + + // write() syscalls + case WRITE: + case WRITEV: + rioop_write(p, t, task); + break; + + // open() and other syscalls which may creat + case OPEN: + case OPEN_AT: + rioop_open(p, t, task, -1); + break; + case CREAT: + // A call to crat() is equivalent to calling open() with flags.. + rioop_open(p, t, task, O_CREAT|O_WRONLY|O_TRUNC); + break; + case MKDIR: + case MKDIR_AT: + rioop_mkdir(p, t, task); + break; + + // rename() syscalls + case RENAME: + case RENAME_AT: + case RENAME_AT2: + rioop_rename(p, t, task); + break; + + // close() and unlink() syscalls + case CLOSE: + rioop_close(p, t, task); + break; + case UNLINK: + case UNLINK_AT: + rioop_unlink(p, t, task); + break; + case RMDIR: + rioop_rmdir(p, t, task); + break; + + // sync() syscalls + case FSYNC: + rioop_fsync(p, t, task); + break; + case FDATASYNC: + rioop_fdatasync(p, t, task); + break; + case SYNC: + case SYNCFS: + case SYNC_FILE_RANGE: + //Error("op(%d) not implemented", op); + break; + + // Other syscalls + case FCNTL: + rioop_fcntl(p, t, task); + break; + case GETDENTS: + rioop_getdents(p, t, task); + break; + case LSEEK: + rioop_lseek(p, t, task); + break; + + // chmod() syscalls + case CHMOD: + rioop_chmod(p, t, task); + break; + case FCHMOD: + rioop_fchmod(p, t, task); + break; + + // chown() syscalls + case CHOWN: + rioop_chown(p, t, task); + break; + case FCHOWN: + case FCHOWNAT: + rioop_fchown(p, t, task); + break; + case LCHOWN: + rioop_lchown(p, t, task); + break; + + // Meta operations (I/O replay internal use only). + case META_EXIT_GROUP: + break; + case META_TIMELINE: + break; + + default: + Error("op(%d) not implemented", op); + break; + } +} + +void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + struct stat buf; + stat(path, &buf); +} + +void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + struct stat buf; + fstat(vfd->fd, &buf); +} + +void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + _Init_path2(4); + rename(path, path2); +} + +void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_bytes(4); + _Init_virtfd; + + char *buf = Calloc(bytes+1, char); + read(vfd->fd, buf, bytes); + free(buf); +} + +void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_bytes(4); + _Init_virtfd; + + char *buf = Calloc(bytes+1, char); + sprintf(buf, "%ld", task->lineno); + Fill_with_stuff(buf, bytes); + if (vfd->fd == 0) { + Debug("%d %d %ld", vfd->fd, vfd->debug, task->lineno); + _Error("ERROR"); + } + write(vfd->fd, buf, bytes); + free(buf); +} + +void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_) +{ + _Init_fd(3); + _Init_path(4); + _Init_flags(6); + + // Special case as this is creat() now + if (flags_ != -1) + flags = flags_; + + bool directory = Has(flags, O_DIRECTORY); + + if (fd > 0) { + if (directory) { + // We can not open a directory via open() otherwise! + flags &= (O_RDONLY & ~(O_RDWR|O_WRONLY|O_CREAT)); + } else { + // We don't want to open the file in read only mode. + // SystemTap could have skipped syscalls to fcntl or open + flags &= ~O_RDONLY; + } + // flags |= O_DIRECT|O_SYNC; + flags &= ~O_EXCL; + } + + int ret = open(path, flags, S_IRWXU|S_IRWXG|S_IRWXO); + + if (fd < 0 && ret > 0) { + close(ret); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|open+close|%s|\n", path); + fflush(t->rthread_fd); +#endif + } + + if (fd > 0 && ret > 0) { + vfd_s *vfd = vfd_new(ret, fd, path); + amap_set(p->fds_map, fd, vfd); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|open|%s|\n", path); + fflush(t->rthread_fd); +#endif + } +} + +void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + + amap_unset(p->fds_map, fd); + if (vfd->dirfd) { + closedir(vfd->dirfd); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|closedir|%s|\n", vfd->path); + fflush(t->rthread_fd); +#endif + } else { + close(vfd->fd); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|close|%s|\n", vfd->path); + fflush(t->rthread_fd); +#endif + } + vfd_destroy(vfd); +} + +void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + + // getdents expects a dirfd + DIR *dirfd = fdopendir(vfd->fd); + if (dirfd) { + vfd->dirfd = dirfd; + readdir(dirfd); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|fdopendir|%s|\n", vfd->path); + fflush(t->rthread_fd); +#endif + } +} + +void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + mkdir(path, S_IRWXU|S_IRWXG|S_IRWXO); +} + +void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + unlink(path); +} + +void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + rmdir(path); +} + +void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_bytes(6); + _Init_virtfd; + lseek(vfd->fd, bytes, SEEK_SET); +} + +void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + fsync(vfd->fd); +} + +void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + fdatasync(vfd->fd); +} + +void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_cmd(4); + _Init_arg(5); + _Init_virtfd; + + switch (cmd) { + case F_GETFD: + case F_GETFL: + fcntl(vfd->fd, cmd); + break; + case F_SETFD: + case F_SETFL: + fcntl(vfd->fd, cmd, arg); + break; + default: + break; + } +} + +void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + chmod(path, S_IRWXU|S_IRWXG|S_IRWXO); +} + +void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + fchmod(vfd->fd, S_IRWXU|S_IRWXG|S_IRWXO); +} + +void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + rworker_s *w = t->worker; + options_s *opts = w->opts; + struct passwd *pwd = getpwnam(opts->user); + chown(path, pwd->pw_uid, -1); +} + +void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + rworker_s *w = t->worker; + options_s *opts = w->opts; + struct passwd *pwd = getpwnam(opts->user); + fchown(vfd->fd, pwd->pw_uid, -1); +} + +void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + rworker_s *w = t->worker; + options_s *opts = w->opts; + struct passwd *pwd = getpwnam(opts->user); + lchown(path, pwd->pw_uid, -1); +} + diff --git a/ioreplay/src/replay/rioop.h b/ioreplay/src/replay/rioop.h new file mode 100644 index 0000000..4db4284 --- /dev/null +++ b/ioreplay/src/replay/rioop.h @@ -0,0 +1,54 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RIOOP_H +#define RIOOP_H + +#include "../defaults.h" +#include "../utils/futils.h" +#include "../opcodes.h" +#include "rprocess.h" +#include "rthread.h" + +/** + * @brief Replays the responsible I/O operation of a given task + * + * @param p The virtual replay process object + * @param t The thread object + * @param task The replay task object + */ +void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task); + +void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_); +void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task); + +#endif // RIOOP_H diff --git a/ioreplay/src/replay/rprocess.c b/ioreplay/src/replay/rprocess.c new file mode 100644 index 0000000..4efd835 --- /dev/null +++ b/ioreplay/src/replay/rprocess.c @@ -0,0 +1,34 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rprocess.h" + +rprocess_s* rprocess_new(const int pid, amap_s *fds_map) +{ + rprocess_s *p = Malloc(rprocess_s); + + p->fds_map = fds_map; + p->pid = pid; + p->terminate = 0; + p->lineno = 0; + + return p; +} + +void rprocess_destroy(rprocess_s *p) +{ + if (!p) + return; + free(p); +} diff --git a/ioreplay/src/replay/rprocess.h b/ioreplay/src/replay/rprocess.h new file mode 100644 index 0000000..739dd89 --- /dev/null +++ b/ioreplay/src/replay/rprocess.h @@ -0,0 +1,40 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RPROCESS_H +#define RPROCESS_H + +#include "../datas/hmap.h" +#include "../datas/amap.h" +#include "../defaults.h" +#include "rthread.h" + +/** + * @brief The virtual replay process object definition + * + * This defines a virtual process in replay context. + */ +typedef struct rprocess_s_ { + int terminate; /**< Indicates whether the worker is terminating or not */ + int rworker_num; /**< The worker number of the responsible worker */ + int pid; /**< The virtual process ID */ + unsigned long lineno; /**< Holding the current .replay line number */ + bool initm; /**< Indicates whether ioreplay is in init mode or not */ + amap_s *fds_map; /**< Holding all file descriptors */ +} rprocess_s; + +rprocess_s* rprocess_new(const int pid, amap_s *fds_map); +void rprocess_destroy(rprocess_s* p); + +#endif // RPROCESS_H diff --git a/ioreplay/src/replay/rstats.c b/ioreplay/src/replay/rstats.c new file mode 100644 index 0000000..c3e6e38 --- /dev/null +++ b/ioreplay/src/replay/rstats.c @@ -0,0 +1,108 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rstats.h" + +#include <sys/types.h> + +rstats_s* rstats_new(options_s *opts) +{ + rstats_s *s = Malloc(rstats_s); + + s->opts = opts; + s->loadavg_high = 0; + s->ioops = 0; + s->duration = 0; + s->time_ahead = -1; + + if (opts->stats_file) + s->stats_fd = Fopen(opts->stats_file, "w"); + else + s->stats_fd = stdout; + + return s; +} + +void rstats_destroy(rstats_s *s) +{ + if (s->stats_fd != stdout) + fclose(s->stats_fd); + + free(s); +} + +rworker_stats_s* rworker_stats_new_mmap(options_s *opts) +{ + // Share this object between processes, so that the stats cann be + // collected by the master process! + rworker_stats_s *s = Mmapshared(rworker_stats_s); + + s->loadavg_high = 0; + s->ioops = 0; + s->time_ahead = -1; + + return s; +} + +void rworker_stats_destroy(rworker_stats_s *s) +{ + munmap(s, sizeof(rworker_stats_s)); +} + + +void rstats_start(rstats_s* s) +{ + gettimeofday(&s->start_time, NULL); +} + +void rstats_stop(rstats_s* s) +{ + gettimeofday(&s->end_time, NULL); + s->duration= ((s->end_time.tv_sec - s->start_time.tv_sec) * 1000 + + (s->end_time.tv_usec - s->start_time.tv_usec) / 1000) / 1000; + +} + +void rstats_add_from_worker(rstats_s* s, rworker_stats_s* w) +{ + if (s->loadavg_high < w->loadavg_high) + s->loadavg_high = w->loadavg_high; + + if (s->time_ahead == -1 || s->time_ahead > w->time_ahead) + s->time_ahead = w->time_ahead; + + s->ioops += w->ioops; +} + +void rstats_print(rstats_s* s) +{ + options_s *opts = s->opts; + + if (opts->stats_file) { + Put("Writing stats to '%s'", opts->stats_file); + } + + fprintf(s->stats_fd, "Stats of test '%s':\n", opts->name); + fprintf(s->stats_fd, "\tNum workers: %d\n", opts->num_workers); + fprintf(s->stats_fd, "\tThreads per worker: %d\n", opts->num_threads_per_worker); + fprintf(s->stats_fd, "\tThreads total: %d\n", + opts->num_threads_per_worker * opts->num_workers); + fprintf(s->stats_fd, "\tHighest loadavg: %.2f\n", s->loadavg_high); + fprintf(s->stats_fd, "\tPerformed ioops: %ld\n", s->ioops); + if (s->duration > 0) + fprintf(s->stats_fd, "\tAverage ioops/s: %.2f\n", s->ioops/s->duration); + fprintf(s->stats_fd, "\tTime ahead: %lds\n", s->time_ahead/1000); + fprintf(s->stats_fd, "\tTotal time: %.2fs\n", s->duration); +} + diff --git a/ioreplay/src/replay/rstats.h b/ioreplay/src/replay/rstats.h new file mode 100644 index 0000000..1ce3f27 --- /dev/null +++ b/ioreplay/src/replay/rstats.h @@ -0,0 +1,117 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file rstats.h + * @author Paul Buetow + * + * @brief For collecting replay stats + */ + +#ifndef RSTATS_H +#define RSTATS_H + +#include "../defaults.h" +#include "../options.h" + +#include <pthread.h> + +/** + * @brief Definition of the rstats object + * + * Used to store global statistics. + */ +typedef struct rstats_s_ { + double loadavg_high; /**< Highest load average */ + long ioops; /**< Total amount if io operations */ + double duration; /**< Duration of the test */ + long time_ahead; /**< Time ahead of the original speed */ + struct timeval start_time; /**< Start time of the test */ + struct timeval end_time; /**< End time of the test */ + options_s *opts; /**< The I/O replay options object */ + FILE *stats_fd; /**< The file descriptor for writing the stats */ +} rstats_s; + +/** + * @brief Definition of the per worker stats object + * + * Used to store per worker process I/O stats + */ +typedef struct rworker_stats_s_ { + double loadavg_high; /**< Highest amount of io ops per second */ + long ioops; /**< Total amount if io operations */ + long time_ahead; /**< Time ahead of the original speed */ +} rworker_stats_s; + +/** + * @brief Creates a new stats object + * + * @return The new stats object + */ +rstats_s* rstats_new(options_s *opts); + +/** + * @brief Destroys the stats object + * + * @param s The stats object + */ +void rstats_destroy(rstats_s* s); + +/** + * @brief Creates a new per worker stats object + * + * The memory is mapped into shared memory so it can be shared across multiple + * processes. + * + * @return The new stats object + */ +rworker_stats_s* rworker_stats_new_mmap(); + +/** + * @brief Destroys the per worker stats object + * + * @param s The stats object + */ +void rworker_stats_destroy(rworker_stats_s* s); + +/** + * @brief Starts the stats + * + * @param s The stats object + */ +void rstats_start(rstats_s* s); + +/** + * @brief Finalises the stats + * + * @param s The stats object + */ +void rstats_stop(rstats_s* s); + +/** + * @brief Prints the stats + * + * @param s The stats object + */ +void rstats_print(rstats_s* s); + +/** + * @brief Adds per worker stats to the global stats object + * + * @param s The global stats object + * @param w The worker stats object + */ +void rstats_add_from_worker(rstats_s* s, rworker_stats_s* w); + +#endif diff --git a/ioreplay/src/replay/rtask.c b/ioreplay/src/replay/rtask.c new file mode 100644 index 0000000..b1afb92 --- /dev/null +++ b/ioreplay/src/replay/rtask.c @@ -0,0 +1,50 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rtask.h" + +#include "rthread.h" +#include "rworker.h" + +rtask_s* rtask_new() +{ + rtask_s *task = Malloc(rtask_s); + + *task = (rtask_s) { + .worker = NULL, .process = NULL + }; + task->line[0] = '\0'; + +#ifdef THREAD_DEBUG + task->clone = NULL; +#endif + + return task; +} + +void rtask_destroy(rtask_s *task) +{ + if (task) + free(task); +} + +void rtask_update(rtask_s *task, void *worker, void *process, char *line, + const long lineno, const long vsize) +{ + task->worker = worker; + task->process = process; + task->lineno = lineno; + task->vsize = vsize; + strcpy(task->line, line); +} diff --git a/ioreplay/src/replay/rtask.h b/ioreplay/src/replay/rtask.h new file mode 100644 index 0000000..35c5714 --- /dev/null +++ b/ioreplay/src/replay/rtask.h @@ -0,0 +1,69 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RTASK_H +#define RTASK_H + +#include "../defaults.h" + +/** + * @brief The replay task definition + * + * The rtask holds all possible variables required to process a particular + * .replay line and to replay the corresponding I/O operation. + */ +typedef struct rtask_s_ { + void *worker; /* The responsible worker object */ + void *process; /* The responsible process object */ + unsigned long lineno; /**< The current line number */ + unsigned long vsize; /**< The vsize */ + char *toks[MAX_TOKENS+1]; /**< The tokens parsed from the .replay line */ + char line[MAX_LINE_LEN]; /**< The remaining part of the .replay line */ +#ifdef RTASK_DEBUG + char *clone; /**< Used for debug purposes only */ +#endif +} rtask_s; + +/** + * @brief Creates a new thread task object + * + * This function creates a new thread task object. Such a task object is used + * by the worker to hand over I/O tasks to the corresponding threads. The + * actual I/O work is performed by the threads then. + * + * @return The new thread task object + */ +rtask_s* rtask_new(); + +/** + * @brief Destroys the replay task object + * + * @param t The thread task object to be destroyed + */ +void rtask_destroy(rtask_s* t); + +/** + * @brief Updates a reused/recycle task object + * + * @param task The task object to be updated + * @param worker The responsibe worker object + * @param process The responsible process object + * @param line The remaining line of the .replay file + * @param lineno The current line number of the .replay file + * @param vsize The vsize/path id + */ +void rtask_update(rtask_s *task, void *worker, void *process, char *line, + const long lineno, const long vsize); + +#endif // RTASK_H diff --git a/ioreplay/src/replay/rthread.c b/ioreplay/src/replay/rthread.c new file mode 100644 index 0000000..55364ec --- /dev/null +++ b/ioreplay/src/replay/rthread.c @@ -0,0 +1,216 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rthread.h" + +#include <sys/types.h> + +#include "rworker.h" +#include "rprocess.h" + +#include "rioop.h" + +#ifdef THREAD_DEBUG +/** + * @brief For debugging purposes only + * + * @param t The responsible thread object + */ +static void _rthread_init_log(rthread_s *t) +{ + rworker_s *w = t->worker; + char *rthread_log = Calloc(1024, char); + snprintf(rthread_log, 1023, "/tmp/ioreplay/worker%d.thread%ld.debuglog", + w->rworker_num, (long)pthread_self()); + + ensure_dir_exists("/tmp/ioreplay"); + t->rthread_fd = Fopen(rthread_log, "a"); + + free(rthread_log); + fprintf(t->rthread_fd, "%ld: DEBUG: Created thread log\n", t->tid); +} +#endif + +void rthread_process_task(rthread_s* t, rtask_s *task, + pid_t pthread_id) +{ + char *next = task->line; + rworker_s *w = (rworker_s*) task->worker; + + // Tokenize the remaining elements of the line. + int ntoks = 0; + char *saveptr; + char *tok = strtok_r(next, "|", &saveptr); + + while (tok) { + if (ntoks > MAX_TOKENS) { + Error("worker(%d) pthread(%d): lineno:%lu, missing newline?", + w->rworker_num, pthread_id, task->lineno); + } + task->toks[ntoks++] = tok; + tok = strtok_r(NULL, "|", &saveptr); + } + // NULL marker (no more token from here) + task->toks[ntoks] = NULL; + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld(%ld): %s", + t->tid, (long)pthread_self(), task->clone); + fflush(t->rthread_fd); + free(task->clone); + task->clone = NULL; +#endif +#ifndef NO_RIOOP + // Perform the corresponding I/O operation! + rioop_run(task->process, t, task); +#endif + + // Make the task object recyclable/reusable + pthread_mutex_lock(&w->task_buffer_mutex); + if (!rbuffer_insert(w->task_buffer, task)) + // We can't recycle the task object if the buffer is full! + rtask_destroy(task); + pthread_mutex_unlock(&w->task_buffer_mutex); +} + +void *rthread_pthread_start(void *data) +{ + rthread_s* t = (rthread_s*) data; + rworker_s *w = t->worker; + rtask_s *task = NULL; + pid_t pthread_id = pthread_self(); + +#ifdef THREAD_DEBUG + _rthread_init_log(t); +#endif + + do { + while (!rbuffer_has_next(t->tasks) && !t->terminate) + usleep(100); + + while ((task = rbuffer_get_next(t->tasks)) != NULL) + rthread_process_task(t, task, pthread_id); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld: DEBUG: Idling\n", t->tid); + fflush(t->rthread_fd); +#endif + + // Tell rworker_s that thread is not doing any work! + int inserted = false; + while (!inserted && !t->terminate) { + if (rbuffer_has_next(t->tasks)) + break; + + usleep(1000); + + if (rbuffer_has_next(t->tasks)) + break; + + // Make the rthread reusable, he is without any tasks + // for some time. + pthread_mutex_lock(&w->rthread_buffer_mutex); + inserted = rbuffer_insert(w->rthread_buffer, t); + pthread_mutex_unlock(&w->rthread_buffer_mutex); + } + +#ifdef THREAD_DEBUG + if (inserted) { + fprintf(t->rthread_fd, "%ld: DEBUG: Added to thread buffer\n", + t->tid); + } else { + fprintf(t->rthread_fd, "%ld: DEBUG: Idling thread recovered\n", + t->tid); + } + fflush(t->rthread_fd); +#endif + + } while (!t->terminate); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld: DEBUG: Terminating\n", t->tid); + fflush(t->rthread_fd); +#endif + + // Process the very last tasks + while (NULL != (task = rbuffer_get_next(t->tasks))) + rthread_process_task(t, task, pthread_id); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld: DEBUG: Done terminating\n", t->tid); + fflush(t->rthread_fd); +#endif + + return NULL; +} + +rthread_s* rthread_new(const long tid, void *worker) +{ + rthread_s *t = Malloc(rthread_s); + rworker_s *w = worker; + + t->single_threaded = w->opts->num_threads_per_worker == 1; + t->tasks = rbuffer_new(TASK_BUFFER_PER_THREAD); + t->terminate = false; + t->worker = worker; + rthread_update(t, tid); + + if (t->single_threaded) { +#ifdef THREAD_DEBUG + _rthread_init_log(t); +#endif + return t; + } + + start_pthread(&t->pthread, rthread_pthread_start, (void*)t); + return t; +} + +long rthread_update(rthread_s *t, const long tid) +{ + long prev_tid = t->tid; + t->tid = tid; + + return prev_tid; +} + +void rthread_destroy(rthread_s *t) +{ + if (rbuffer_has_next(t->tasks)) { + Error("Didn't expect to have any tasks left!"); + } + rbuffer_destroy(t->tasks); + +#ifdef THREAD_DEBUG + if (t->rthread_fd) + fclose(t->rthread_fd); +#endif + + free(t); +} + +bool rthread_insert_task(rthread_s* t, rtask_s* task) +{ + if (t->single_threaded) { + rthread_process_task(t, task, pthread_self()); + return true; + } + return rbuffer_insert(t->tasks, task); +} + +void rthread_terminate(rthread_s* t) +{ + t->terminate = true; + pthread_join(t->pthread, NULL); +} diff --git a/ioreplay/src/replay/rthread.h b/ioreplay/src/replay/rthread.h new file mode 100644 index 0000000..9971e49 --- /dev/null +++ b/ioreplay/src/replay/rthread.h @@ -0,0 +1,123 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file rthread.h + * @author Paul Buetow + * + * @brief The replay thread definitiion + */ + +#ifndef RTHREAD_H +#define RTHREAD_H + +#include "../defaults.h" +#include "../datas/rbuffer.h" +#include "../datas/amap.h" +#include "../vfd.h" +#include "rtask.h" + +#include <pthread.h> + +/** + * @brief Definition of a worker thread + * + * Every worker utilises a set of worker threads in order to parallelise the + * replaying of the I/O! Every thread comes with its own task queue. It is + * filled by the repsonsible worker. + * + * The user can specify the max amount of threads per worker per -t command + * line switch. + */ +typedef struct rthread_s_ { + void *worker; /**< The responsible worker object */ + long tid; /**< The virtual thread id */ + rbuffer_s* tasks; /**< Holds all outstanding tasks */ + bool terminate; /**< True if thread shall terminate */ + bool single_threaded; /**< Worker is single threaded or not */ + pthread_t pthread; /**< We run the tasks in concurrent pthreads */ +#ifdef RTHREAD_DEBUG + FILE *rthread_fd; /**< Used for debugging purposes only */ +#endif +} rthread_s; + +/** + * @brief Creates a new thread object + * + * @param tid The thread ID + * @param worker The worker object managing this thread + * @return The new thread object + */ +rthread_s* rthread_new(const long tid, void *worker); + +/** + * @brief Updates a thread object after recycling it + * + * @param t The thread object + * @param tid The new thread ID + */ +long rthread_update(rthread_s *t, const long tid); + +/** + * @brief Terminates the thread + * + * This function waits (via join) for the pthread to complete all its + * current tasks from the queue. + * + * @param t The thread object + */ +void rthread_terminate(rthread_s* t); + +/** + * @brief Destroys the thread object + * + * @param t The thread object + */ +void rthread_destroy(rthread_s* t); + +/** + * @brief Inserts a task into the threads work queue + * + * Inserts a task into the threads work queue. We use an atomic ring buffer + * data structure for the work queue. The ring buffer does not require any + * mutex locks. + * + * @param t The thread object + * @param task The task to be inserted + * @return Returns true on success, returns false if the task queue is full + */ +bool rthread_insert_task(rthread_s* t, rtask_s* task); + +/** + * @brief Used by the pthread to process a task + * + * In this function the pthread will attempt to process a task. It extracts all + * required information from the task object and invokes the corresponding I/O + * syscalls. + * + * @param t The responsible thread object + * @param task The task object + * @param pthread_id The current pthread id + */ +void rthread_process_task(rthread_s* t, rtask_s *task, pid_t pthread_id); + +/** + * @brief The entry function for the pthreads + * + * @param data The data structure passed to the pthread + * @return The exit code of the pthread. + */ +void *rthread_pthread_start(void *data); + +#endif // RTHREAD_H diff --git a/ioreplay/src/replay/rworker.c b/ioreplay/src/replay/rworker.c new file mode 100644 index 0000000..5a50ada --- /dev/null +++ b/ioreplay/src/replay/rworker.c @@ -0,0 +1,360 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rworker.h" + +#include "../datas/stack.h" +#include "rprocess.h" +#include "rthread.h" + +#define _Compute_current_time(now) \ + (now.tv_sec - start_time.tv_sec) * 1000 \ + + (now.tv_usec - start_time.tv_usec) / 1000 + + +/** + * @brief A callback helper function for destroying all virtual process objects + * + * @param data The process object. + */ +static void _rprocess_destroy_cb(void *data) +{ + rprocess_destroy(data); +} + +rworker_s* rworker_new(const int rworker_num, amap_s *fds_map, + const long num_vsizes, const long num_pids, + options_s *opts, rworker_stats_s *worker_stats) +{ + rworker_s *w = Malloc(rworker_s); + +#ifdef THREAD_DEBUG + char *rworker_log = Calloc(1024, char); + snprintf(rworker_log, 1023, "/tmp/ioreplay/_worker%d.debuglog", + rworker_num); + + w->rworker_fd = Fopen(rworker_log, "a"); + free(rworker_log); + fprintf(w->rworker_fd, "DEBUG: Started worker\n"); +#endif + + w->rworker_num = rworker_num; + w->opts = opts; + w->fds_map = fds_map; + + w->rprocess_map = amap_new(num_pids); + w->rthread_map = amap_new(num_vsizes); + w->task_buffer = rbuffer_new(opts->num_threads_per_worker + *TASK_BUFFER_PER_THREAD); + w->rthread_buffer = rbuffer_new(opts->num_threads_per_worker); + w->worker_stats = worker_stats; + + // Attach a cleanup callback function to the worker map. + w->rprocess_map->data_destroy = _rprocess_destroy_cb; + + pthread_mutex_init(&w->rthread_buffer_mutex, NULL); + pthread_mutex_init(&w->task_buffer_mutex, NULL); + + // TODO: Check in the program whether the ulimit is high enough + // or not! (ulimit -n) + + return w; +} + +/** + * @brief Destroys the object + * + * Destroys the worker object (frees all memory allocated by the worker) + * + * @param w The worker object + */ +void rworker_destroy(rworker_s *w) +{ + if (!w) + return; + + if (w->rprocess_map) + amap_destroy(w->rprocess_map); + if (w->rthread_map) + amap_destroy(w->rthread_map); + + if (w->task_buffer) { + rtask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(w->task_buffer))) + rtask_destroy(task); + rbuffer_destroy(w->task_buffer); + } + + if (w->rthread_buffer) + rbuffer_destroy(w->rthread_buffer); + + pthread_mutex_destroy(&w->task_buffer_mutex); + pthread_mutex_destroy(&w->rthread_buffer_mutex); + +#ifdef THREAD_DEBUG + if (w->rworker_fd) + fclose(w->rworker_fd); +#endif + + free(w); +} + +status_e rworker_process_lines(rworker_s* w, const long num_lines) +{ + Out("worker(%d): Starting to process replay lines\n", w->rworker_num); + + options_s *opts = w->opts; + FILE *replay_fd = Fopen(opts->replay_file, "r"); + + // Drop root privileges, otherwise we may overwrite other system + // files by accident in case of a bug or user error! + drop_root(opts->user); + + // Variables required for the time based caluclations + struct timeval now, start_time; + long current_time = 0, stats_time = 0; + gettimeofday(&start_time, NULL); + + // Helper variables required for reading lines + char *line = NULL; + char *next = NULL, *next2 = NULL; + size_t len = 0, read = 0; + + // Helpers required for threading + rthread_s *t = NULL; + stack_s *all_threads = stack_new(); + rworker_stats_s *s = w->worker_stats; + + // More helper variables + //unsigned long lineno = 0, stats_ioop = 0, vsize_id = 0; + unsigned long lineno = 0, vsize_id = 0; + long pid = -1, time = -1; + + // Process the .replay file line by line. + while ((read = getline(&line, &len, replay_fd)) != -1) { + lineno++; + + if (read >= MAX_LINE_LEN) { + Error("line:%lu Exceeded max line len", lineno); + } + + // If the line begins with #: Ignore that line, it contains + // debug or meta information or comments. + + if (line[0] == '#') { + if (line[1] == 'I') { + // We stop replaying I/O once we reach the line '#INIT' + // which incitates the begin of the INIT section. + break; + } + continue; + } + +#ifdef THREAD_DEBUG + char *clone = Clone(line); +#endif + + next = strchr(line, '|'); + Error_if(!next, "lineno:%ld Could not parse time from input file", + lineno); + next[0] = '\0'; + next++; + time = atol(line); + + next2 = strchr(next, '|'); + Error_if(!next2, "Could not parse vsize_id from input file"); + next2[0] = '\0'; + next2++; + vsize_id = atol(next); + + // This worker is not responsible for this line, skip it! + if ((vsize_id % opts->num_workers) != w->rworker_num) { +#ifdef THREAD_DEBUG + free(clone); +#endif + continue; + } + + next = strchr(next2, '|'); + Error_if(!next, "Could not parse PID from input file"); + next[0] = '\0'; + next++; + pid = atol(next2); + + gettimeofday(&now, NULL); + current_time = _Compute_current_time(now); + + // Check whether the user specified a replay speed factor. If so, we + // may need to throttle down a bit. + + if (opts->speed_factor) { + s->time_ahead = time / opts->speed_factor - current_time; + if (s->time_ahead > 0) + usleep(s->time_ahead*1000); + + } else { + s->time_ahead = time - current_time; + } + + // Get the responsible process object. The process object holds data + // structures usually found in a Linux process, e.g. a table of open + // file descriptors. + + rprocess_s *p = amap_get(w->rprocess_map, pid); + if (p == NULL) { + p = rprocess_new(pid, w->fds_map); + amap_set(w->rprocess_map, pid, p); + } + p->lineno = lineno; + + if (opts->num_threads_per_worker == 1) { + // Single threaded mode? + if (!t) + t = rthread_new(vsize_id, w); + else + rthread_update(t, vsize_id); + + } else { + t = amap_get(w->rthread_map, vsize_id); + } + + if (t == NULL) { + + // First try to recycle an old (likely unused) thread + if (NULL != (t = rbuffer_get_next(w->rthread_buffer))) { + rthread_update(t, vsize_id); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Reused an idling thread\n"); + fflush(w->rworker_fd); +#endif + + } else if (opts->num_threads_per_worker <= all_threads->size) { + // Reached max threads, waiting until one becomes available + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Reached max threads\n"); + fflush(w->rworker_fd); +#endif + while (NULL == (t = rbuffer_get_next(w->rthread_buffer))) + usleep(1000); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Reused an idling thread\n"); + fflush(w->rworker_fd); +#endif + + rthread_update(t, vsize_id); + + } else { + t = rthread_new(vsize_id, w); + + // We hold a pointer to all created threads in a stack. This + // stack is later used to terminate/join all therads. + stack_push(all_threads, t); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Created a new thread\n"); + fflush(w->rworker_fd); +#endif + } + + amap_set(w->rthread_map, vsize_id, t); + } + + // Create a new task for the thread. The task contains all required + // information to run an I/O operation. However, first try to + // reuse/recycle a task object! If there is no such, create a new one. + + rtask_s *task = rbuffer_get_next(w->task_buffer); + if (!task) + task = rtask_new(); + rtask_update(task, w, p, next, lineno, vsize_id); + s->ioops++; + + +#ifdef THREAD_DEBUG + task->clone = clone; + fprintf(w->rworker_fd, "DEBUG: Inserting new task\n"); + fflush(w->rworker_fd); +#endif + + // Insert that task to a ring buffer to pass it to the pthread without + // much synchronisation overhead! + + while (!rthread_insert_task(t, task)) + // The ring buffer is full. This may happen if the pthread didn't + // manage to process tasks fast enough. re-try after a short period! + usleep(1000); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Task inserted\n"); + fflush(w->rworker_fd); +#endif + + // The worker prints out stats every 3s + if (current_time - stats_time > 3000) { + // IDEA: Maybe refactor this block to be implemented in rstats.c + + double loadavg = get_loadavg(); + + // Determines whether we replay the I/O faster or slower than + // original speed! + char *a_b = s->time_ahead >= 0 ? "ahead" : "behind"; + + Put("worker(%d): threads:%ld %s:%lds progress:%0.2f%% " + "loadavg:%0.2f", + w->rworker_num, all_threads->size, a_b, Abs(s->time_ahead/1000), + Perc(lineno,num_lines), loadavg); + + stats_time = current_time; + //stats_ioop = lineno; + + if (s->loadavg_high < loadavg) + s->loadavg_high = loadavg; + } + } + + Put("worker(%d): Waiting for all threads to finish business...", + w->rworker_num); + + // This will wait (join) all threads one after another until all threads + // have finished their work and have terminated. + + while (!stack_is_empty(all_threads)) { + rthread_s *t = stack_pop(all_threads); + rthread_terminate(t); + rthread_destroy(t); + } + stack_destroy(all_threads); + + // Collect some stats last time + double loadavg = get_loadavg(); + if (s->loadavg_high < loadavg) + s->loadavg_high = loadavg; + + gettimeofday(&now, NULL); + current_time = _Compute_current_time(now); + if (opts->speed_factor) { + s->time_ahead = time / opts->speed_factor - current_time; + } else { + s->time_ahead = time - current_time; + } + + + Put("worker(%d): All threads terminated!", w->rworker_num); + fclose(replay_fd); + + return SUCCESS; +} diff --git a/ioreplay/src/replay/rworker.h b/ioreplay/src/replay/rworker.h new file mode 100644 index 0000000..26a1300 --- /dev/null +++ b/ioreplay/src/replay/rworker.h @@ -0,0 +1,82 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RWORKER_H +#define RWORKER_H + +#include <pthread.h> + +#include "../datas/amap.h" +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "../options.h" +#include "rstats.h" + +/** + * @brief Represents a worker process. + * + * This represents an I/O replay worker process. The user can specify the + * amount of worker processes via the -p command line switch. This is not + * to confuse with rprocess_s, which represents an original captured process + * and we now want to replay the I/O for! + */ +typedef struct { + int rworker_num; /**< The current worker ID */ + amap_s* fds_map; /**< Holding all file descriptors */ + amap_s* rprocess_map; /**< Holding all processes handled by this worker */ + amap_s* rthread_map; /**< Holding all threads handled by this worker */ + rbuffer_s *task_buffer; /**< Buffering thread tasks to be reused */ + pthread_mutex_t task_buffer_mutex; /**< To sync access to task_buffer */ + rbuffer_s *rthread_buffer; /**< Buffering idle threads to be reused */ + pthread_mutex_t rthread_buffer_mutex; /**< Sync access to rthread_buffer */ + options_s *opts; /**< To synchronise access to rthread_buffer */ + rworker_stats_s *worker_stats; /**< Object holding per worker statistics */ +#ifdef RTHREAD_DEBUG + FILE *rworker_fd; /**< For debugging purposes only */ +#endif +} rworker_s; + +/** + * @brief Creates a new worker object + * + * @param rworker_num The worker number + * @param fds_map A map of all virtual file descriptor objects + * @param num_vsizes The amount of virtual sizes/total file paths of the test + * @param num_pids The total amount of virtual process IDs used in this test + * @param opts A pointer to the options object + * @param worker_stats A pointer to the worker stats object + + * @return The new worker object + */ +rworker_s* rworker_new(const int rworker_num, amap_s *fds_map, + const long num_vsizes, const long num_pids, + options_s* opts, rworker_stats_s *worker_stats); + +/** + * @brief Destroys a worker object + * + * @param w The worker object to be destroyed + */ +void rworker_destroy(rworker_s* w); + +/** + * @brief Makes the worker to process all .replay lines + * + * @param w The responsible worker object + * @param num_lines The total amount of I/O op lines in the .replay file + * @return SUCCESS if everything went fine + */ +status_e rworker_process_lines(rworker_s* w, const long num_lines); + +#endif // RWORKER_H diff --git a/ioreplay/src/utests.c b/ioreplay/src/utests.c new file mode 100644 index 0000000..5812a66 --- /dev/null +++ b/ioreplay/src/utests.c @@ -0,0 +1,30 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utests.h" + +#include "datas/amap.h" +#include "datas/hmap.h" +#include "datas/list.h" +#include "datas/rbuffer.h" + +void utests_run() +{ + fprintf(stderr, "Running unit tests\n"); + amap_test(); + hmap_test(); + list_test(); + rbuffer_test(); + fprintf(stderr, "Great success, run all unit tests without any errors!\n"); +} diff --git a/ioreplay/src/utests.h b/ioreplay/src/utests.h new file mode 100644 index 0000000..4ad6973 --- /dev/null +++ b/ioreplay/src/utests.h @@ -0,0 +1,25 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef UTESTS_H +#define UTESTS_H + +#include "utils/utils.h" + +/** + * @brief This function runs all currently implemented unit tests + */ +void utests_run(); + +#endif // UTESTS_H diff --git a/ioreplay/src/utils/futils.c b/ioreplay/src/utils/futils.c new file mode 100644 index 0000000..5b35618 --- /dev/null +++ b/ioreplay/src/utils/futils.c @@ -0,0 +1,291 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "futils.h" + +#include <libgen.h> +#include <pwd.h> +#include <unistd.h> +#include <limits.h> + +#include "../macros.h" + +void append_random_to_file(char *path, unsigned long bytes) +{ + char *buf = NULL; + int max_chunk = 50000000; // 50 mebibyetes + FILE *fp = Fopen(path, "a"); + + for (;;) { + if (bytes > max_chunk) { + if (!buf) + buf = Calloc(max_chunk+1, char); + + Fill_with_stuff(buf, max_chunk); + buf[max_chunk] = '\0'; + fprintf(fp, "%s", buf); + bytes -= max_chunk; + + // Print out a dot every time we wrote 'much' data to a file + Out("."); + + } else { + if (!buf) + buf = Calloc(bytes+1, char); + + Fill_with_stuff(buf, bytes); + buf[bytes] = '\0'; + fprintf(fp, "%s", buf); + + break; + } + } + + if (buf) + free(buf); + fclose(fp); +} + +long ensure_dir_exists(const char *path) +{ + long num_dirs_created = 0; + int ret = mkdir_p(path, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH, + &num_dirs_created); + if (ret != 0) { + Errno("Could not create dir '%s'", path); + } + + return num_dirs_created; +} + +void ensure_parent_dir_exists(const char *path) +{ + char *clone = Clone(path); + char *parent = dirname(clone); + + int ret = mkdir_p(parent, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH, NULL); + if (ret != 0) { + Errno("Could not create dir %s", parent); + } + + free(clone); +} + +void ensure_dir_empty(const char *path) +{ + DIR *dh = opendir(path); + + if (!dh) { + Errno("Unable to empty %s", path); + } + + struct dirent *de; + + while ((de = readdir(dh))) { + if (0 == strcmp(de->d_name, ".") || + 0 == strcmp(de->d_name, "..")) + continue; + + char *absolute; + asprintf(&absolute, "%s/%s", path, de->d_name); + + if (is_dir(absolute)) + ensure_dir_empty(absolute); + + if (remove(absolute) == -1) + // Don't throw an error if there is no such file or directory + if (errno != 2) { + Errno("Unable to remove %s", absolute); + } + + free(absolute); + } + + closedir(dh); +} + +int ensure_file_exists(char *path, long *num_dirs_created) +{ + if (is_reg(path)) + return SUCCESS; + + char *dirname = dirname_r(Clone(path)); + *num_dirs_created += ensure_dir_exists(dirname); + free(dirname); + + FILE *fp = fopen(path, "a"); + if (fp) { + // We only need some data, less than 1 block in size, this is answer: + fprintf(fp, "42"); + fclose(fp); + return SUCCESS; + } + + return ERROR; +} + +char* dirname_r(char *path) +{ + int len = strlen(path); + int has = 0; + int i = len-1; + + if (strcmp(path, "..") == 0) { + return path; + } + + if (path[i] == '/') { + // Root directory + if (len == 1) + return path; + + // Remove all trailing / + for (; i >= 0; --i) { + if (path[i] == '/') { + path[i] = '\0'; + has = 1; + } else { + break; + } + } + } + + // Find next / + for (; i >= 0; --i) { + if (path[i] == '/') { + path[i] = '\0'; + has = 1; + break; + } + } + + // If no / + if (has == 0) { + path[0] = '.'; + path[1] = '\0'; + } + + return path; +} + +bool is_dir(const char *path) +{ + struct stat path_stat; + if (stat(path, &path_stat) == 0 && S_ISDIR(path_stat.st_mode)) + return true; + return false; +} + +bool is_reg(const char *path) +{ + struct stat path_stat; + if (stat(path, &path_stat) == 0 && S_ISREG(path_stat.st_mode)) + return true; + return false; +} + +bool exists(const char *path) +{ + struct stat path_stat; + if (stat(path, &path_stat) == 0) + return true; + return false; +} + +int mkdir_p(const char *path, mode_t mode, long *num_dirs_created) +{ + int res = 0; + + if (is_dir(path)) + return 0; + + if (is_reg(path)) + unlink(path); + + char *top = dirname_r(Clone(path)); + if (0 != mkdir_p(top, mode, num_dirs_created)) + goto cleanup; + + if ((mkdir(path, mode) == -1) && (errno != EEXIST)) + res = -1; + + if (res != -1) + *num_dirs_created = *num_dirs_created+1; + +cleanup: + free(top); + + return res; +} + +void cache_file(const char *file) +{ + Out("Caching file %s... it can take a while", file); + FILE *fd = Fopen(file, "r"); + char *line = NULL; + size_t len = 0, read = 0; + + while ((read = getline(&line, &len, fd)) != -1); + fclose(fd); +} + +void drop_caches(void) +{ + Out("Dropping all Linux caches..."); + + if (getuid() != 0) { + Out("\n"); + Error("I need to be root to do this, aborting!"); + } + + // echo 3 > /proc/sys/vm/drop_caches + char *drop_caches = "/proc/sys/vm/drop_caches"; + FILE *fd = Fopen(drop_caches, "w"); + fprintf(fd, "3"); + fclose(fd); + + Put("done"); +} + +void chown_path(const char *user, const char *path) +{ + struct passwd *pwd = getpwnam(user); + if (!pwd) { + Errno("Unable to retrieve information about system user %s!", user); + } + + if (chown(path, pwd->pw_uid, -1) == -1) { + Errno("Could not change ownership of '%s' to '%s'!", path, user); + } +} + +char *absolute_path(const char *path) +{ + if (path[0] == '/') + return Clone(path); + + char cwd[MAX_LINE_LEN]; + getcwd(cwd, sizeof(char)*MAX_LINE_LEN); + + if (!getcwd(cwd, sizeof(cwd))) { + Errno("Could not get current working directory"); + } + + char *absolute = NULL; + if (-1 == asprintf(&absolute, "%s/%s", cwd, path)) { + Error("Could not get absolute path of '%s'", path); + } + + return absolute; +} diff --git a/ioreplay/src/utils/futils.h b/ioreplay/src/utils/futils.h new file mode 100644 index 0000000..9afde1a --- /dev/null +++ b/ioreplay/src/utils/futils.h @@ -0,0 +1,134 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef FUTILS_H +#define FUTILS_H + +#include "../defaults.h" + +/** + * @brief Thread safe version of dirname() + * + * @param path The full file path + * @return The directory path + */ +char* dirname_r(char *path); + +/** + * @brief Ensures that a file exists + * + * @param path The file path + * @param num_dirs_created Holds a count of how many sub dirs have been created + * @return -1 on error, 0 on success. + */ +int ensure_file_exists(char *path, long *num_dirs_created); + +/** + * @brief Checks whether path exists + * + * @param path The path + * @return true if the path exists + */ +bool exists(const char *path); + +/** + * @brief Check if path is a directory + * + * @param path The directory path + * @return true if the path is a directory, false otherwise + */ +bool is_dir(const char *path); + +/** + * @brief Check if path is a regular file + * + * @param path The file path + * @return true if the file at path is a regular fike + */ +bool is_reg(const char *path); + +/** + * @brief Create a directory recursively + * + * @param path The directory path + * @param mode The mode + * @param num_dirs_created Counts how many directories have been created + * @return -1 on error + */ +int mkdir_p(const char *path, mode_t mode, long *num_dirs_created); + +/** + * @brief Appends data to a file + * + * @param path The file path + * @param bytes The amount of bytes + */ +void append_random_to_file(char *path, unsigned long bytes); + +/** + * @brief Ensures that a directory exists + * + * @param path The directory path + * @return The amount of directories created (including parent directories) + */ +long ensure_dir_exists(const char *path); + +/** + * @brief Ensures that a parent directory exists + * + * @param path The directory path + */ +void ensure_parent_dir_exists(const char *path); + +/** + * @brief Ensures that a directory is empty + * + * @param path The directory path + */ +void ensure_dir_empty(const char *path); + +/** + * @brief Loading a file into the file system cache + * + * @param file The path to the file + */ +void cache_file(const char *file); + +/** + * @brief Drop all Linux caches + * + * This function drops all Linux caches, which includes all file + * system caches. + */ +void drop_caches(void); + +/** + * @brief Changes owner of a path + * + * Terminates the process with an error message if failed. + * + * @param user The new owner + * @param path The path + */ +void chown_path(const char *user, const char *path); + +/** + * @brief Retrieves the absolute path of a given path + * + * @param path The path + * @return The absolute path. It must be freed manually. + */ +char *absolute_path(const char *path); + +#endif // FUTILS_H diff --git a/ioreplay/src/utils/utils.c b/ioreplay/src/utils/utils.c new file mode 100644 index 0000000..57d6737 --- /dev/null +++ b/ioreplay/src/utils/utils.c @@ -0,0 +1,152 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utils.h" + +#include <sys/resource.h> +#include <sys/time.h> + +void* notnull(void *p, char *file, int line, int count) +{ + if (p == NULL) { + Errno("%s:%d count:%d Could not allocate memory", file, line, count); + } + return p; +} + + +FILE* fnotnull(FILE *fd, const char *path, char *file, int line) +{ + if (fd == NULL) { + Errno("%s:%d Could not open file '%s'", file, line, path); + } + return fd; +} + +void* mmapok(void *p, char *file, int line) +{ + if (p == MAP_FAILED) { + Errno("%s:%d: Mmap failed", file, line); + } + return p; +} + +char* strtok2_r(char *str, char *delim, char **saveptr) +{ + int len = strlen(delim); + + if (str == NULL) + str = *saveptr; + + char *next = strstr(str, delim); + if (next) { + next[0] = '\0'; + for (int i = 0; i < len; ++i) + next++; + *saveptr = next; + return str; + } + + return NULL; +} + +void chreplace(char *str, char replace, char with) +{ + for (int i = 0; ; ++i) { + if (str[i] == '\0') + break; + if (str[i] == replace) + str[i] = with; + } +} + +void strunquote(char *str) +{ + int len = strlen(str); + + if (str[0] == '"') { + if (str[len-1] == '"') + str[len-1] = '\0'; + for (int i = 1; i < len; ++i) + str[i-1] = str[i]; + } +} + +void drop_root(const char *user) +{ + if (getuid() == 0) { + Put("Dropping root privileges to user %s", user); + + struct passwd *pw = getpwnam(user); + + /* process is running as root, drop privileges */ + if (setgid(pw->pw_gid) != 0) { + Errno("setgid: Unable to drop group privileges!"); + } + if (setuid(pw->pw_uid) != 0) { + Errno("setuid: Unable to drop user privileges!"); + } + } +} + +void get_loadavg_s(char *readbuf) +{ + FILE *fp = Fopen("/proc/loadavg", "r"); + fgets(readbuf, 128, fp); + char *pos = strchr(readbuf, ' '); + pos[0] = '\0'; + fclose(fp); +} + +double get_loadavg() +{ + // Not thread safe, but multi processing safe + static char buf[128]; + get_loadavg_s(buf); + + return atof(buf); +} + +bool is_number(char *str) +{ + for (int i = 0; ; ++i) { + if (str[i] == '\0') + return true; + if (isdigit(str[i]) == 0 && str[i] != '-') + return false; + } + + return true; +} + +void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data) +{ + int rc = pthread_create(thread, NULL, cb, data); + + switch (rc) { + case 0: + break; + case EAGAIN: + Error("Out of resources while creating pthread (%d)", rc); + break; + case EINVAL: + Error("Ivalid settings while creating pthread (%d)", rc); + break; + case EPERM: + Error("No permissions to configure pthread (%d)", rc); + default: + Error("Unknown error while creating pthread (%d)", rc); + break; + } +} diff --git a/ioreplay/src/utils/utils.h b/ioreplay/src/utils/utils.h new file mode 100644 index 0000000..cfe4dbc --- /dev/null +++ b/ioreplay/src/utils/utils.h @@ -0,0 +1,165 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef UTILS_H +#define UTILS_H + +// For asprintf in stdio.h +#define _GNU_SOURCE + +#include <assert.h> +#include <ctype.h> +#include <dirent.h> +#include <errno.h> +#include <fcntl.h> +#include <libgen.h> +#include <linux/types.h> +#include <linux/unistd.h> +#include <pthread.h> +#include <pwd.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <time.h> +#include <unistd.h> + +#include "../macros.h" +#include "../defaults.h" + +/** + * @brief Check whether allocated memory is not NULL + * + * This function is used in conjunction with malloc() and co. It + * introduces an extra sanity check whether the memory could be + * allocated successfully or not. If not it will print out a error + * message stating the position in the source code where the memory + * allocation failed! + * + * @param p The pointer being checked + * @param file The source file name the memory was allocated in + * @param line The source line number the memory was allocated at + * @param count The amount of memory being allocated + * @return The pointer to the allocated memory + */ +void* notnull(void *p, char *file, int line, int count); + +/** + * @brief Check whether opened file handle is not NULL + * + * This function is used in conjunction with fopen(). It + * introduces an extra sanity check whether the file could be + * opened successfully or not. If not it will print out a error + * message stating the position in the source code where the open + * failed! + * + * @param fd The fd stream to be checked. + * @param path The file path opened + * @param file The source file name + * @param line The source line number + * @return The pointer to the allocated memory + */ +FILE* fnotnull(FILE *fd, const char *path, char *file, int line); + +/** + * @brief Check whether allocated memory via mmap is not null + * + * This function is used in conjunction with mmap() and co. It + * introduces an extra sanity check whether the memory could be + * allocated successfully or not. If not it will print out a error + * message stating the position in the source code where the memory + * allocation failed! + * + * @param addr The pointer being checked + * @param file The source file name the memory was allocated in + * @param line The source line number the memory was allocated at + * @return The pointer to the allocated memory + */ +void* mmapok(void *addr, char *file, int line); + +/** + * @brief A version of strtok_r supporting multi char delims + * + * @param str The input string + * @param delim The multi-char delimiter + * @param saveptr A temp storage location + * @return The next match if != NULL + */ +char* strtok2_r(char *str, char *delim, char **saveptr); + +/** + * @brief Replaces a character with another one in a string + * + * @param str The input string + * @param replace The character to be replaced + * @param with The character to replace with + */ +void chreplace(char *str, char replace, char with); + +/** + * @brief Removes quotes from a string + * + * @param str The input sting + */ +void strunquote(char *str); + +/** + * @brief Drop root privileges + * + * @param user The user to switch to + */ +void drop_root(const char *user); + +/** + * @brief Retrieve current 1 min Linux load average + * + * @param readbuf The buffer to store the load average as a string + */ +void get_loadavg_s(char *readbuf); + +/** + * @brief Retrieve current 1 min Linux load average + * + * This function is not thread safe! + * + * @return The 1 minute load average of the system + */ +double get_loadavg(); + +/** + * @brief Check whether a string represents a number + * + * @param str The input string + * @return true if all characters of the input string are a digits + */ +bool is_number(char *str); + +/** + * @brief Wrapper around pthread_create + * + * The wrapper also checks whether the thread has been created successfully + * or not! It will exit the process if not. + * + * @param thread The POSIX thread variable + * @param cb The threadss start callback routine + * @param data A data pointer passed to the thread. + */ +void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data); + +#endif // UTILS_H diff --git a/ioreplay/src/vfd.c b/ioreplay/src/vfd.c new file mode 100644 index 0000000..6e86f61 --- /dev/null +++ b/ioreplay/src/vfd.c @@ -0,0 +1,55 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "vfd.h" + +vfd_s* vfd_new(const int fd, const long mapped_fd, char *path) +{ + vfd_s *vfd = Malloc(vfd_s); + vfd->path = NULL; + vfd->debug = false; + vfd_update(vfd, fd, mapped_fd, path); + + return vfd; +} + +void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path) +{ + vfd->fd = fd; + vfd->dirfd = NULL; + vfd->mapped_fd = mapped_fd; + vfd->offset = 0; + + if (path) + free(vfd->path); + + vfd->path = Clone(path); +} + +void vfd_destroy(vfd_s *vfd) +{ + if (!vfd) + return; + + if (vfd->path) + free(vfd->path); + + free(vfd); +} + +void vfd_print(vfd_s *vfd) +{ + fprintf(stderr, "virtfd(%p) fd:%x mapped_fd:%lx path:%s\n", + (void*)vfd, vfd->fd, vfd->mapped_fd, vfd->path); +} diff --git a/ioreplay/src/vfd.h b/ioreplay/src/vfd.h new file mode 100644 index 0000000..fd0c4fb --- /dev/null +++ b/ioreplay/src/vfd.h @@ -0,0 +1,77 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef VFD_H +#define VFD_H + +#include "opcodes.h" +#include "defaults.h" + +/** + * @brief The virtual file descriptor definition + * + * A virtual file descriptor represents a file descriptor from ioreplay's + * point of view and is being used in various ways to simulate the real I/O + * protocolled to the .capture/.replay files. + * + * Generally speaking I/O replay maps the real FD numbers (the ones logged to + * the .capture file) to virtual FD numbers (a uniqe FD number for every open + * to increase concurrency). + */ +typedef struct vfd_s_ { + int fd; /**< the real fd */ + DIR *dirfd; /**< The real dirfd */ + long mapped_fd; /**< The mapped fd (virtual fd) */ + char *path; /**< The file path belonging to that fd */ + bool free_path; /**< True if path has to be freed or not */ + unsigned long offset; /**< The current virtual file offset in bytes */ + int debug; /**< Used for debugging purposes only */ +} vfd_s; + +/** + * @brief Creates a new virtual file descriptor object + * + * @param fd The file descriptor + * @param mapped_fd The mapped file descriptor + * @param path The path name + * @return The new fd object + */ +vfd_s* vfd_new(const int fd, const long mapped_fd, char *path); + +/** + * @brief Updates the virtfd object + * + * @param vfd The virtfd object + * @param fd The (real) file descriptor + * @param mapped_fd The mapped (virtual) file descriptor + * @param path The path name + * @return The new fd object + */ +void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path); + +/** + * @brief Destroys a file descriptor object + * + * @param vfd The file descriptor object + */ +void vfd_destroy(vfd_s *vfd); + +/** + * @brief Prints the virtual file descriptor + * @param vfd The virtual file descriptor + */ +void vfd_print(vfd_s *vfd); + +#endif // VFD_H + diff --git a/ioreplay/tags b/ioreplay/tags new file mode 100644 index 0000000..cbad51a --- /dev/null +++ b/ioreplay/tags @@ -0,0 +1,661 @@ +!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ +!_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ +!_TAG_PROGRAM_AUTHOR Darren Hiebert /dhiebert@users.sourceforge.net/ +!_TAG_PROGRAM_NAME Exuberant Ctags // +!_TAG_PROGRAM_URL http://ctags.sourceforge.net /official site/ +!_TAG_PROGRAM_VERSION 5.8 // +AMAP_H ./src/datas/amap.h 2;" d +AMAP_MAX_ARRAY_LENGTH ./src/datas/amap.h 6;" d +Abs ./src/macros.h 13;" d +BTREE_H ./src/datas/btree.h 2;" d +CAPTURE_VERSION ./src/defaults.h 7;" d +CHMOD ./src/opcodes.h /^ CHMOD = 100,$/;" e enum:__anon1 +CHOWN ./src/opcodes.h /^ CHOWN = 110,$/;" e enum:__anon1 +CHOWN16 ./src/opcodes.h /^ CHOWN16,$/;" e enum:__anon1 +CLEANUP_H ./src/cleanup/cleanup.h 2;" d +CLOSE ./src/opcodes.h /^ CLOSE = 50,$/;" e enum:__anon1 +CREAT ./src/opcodes.h /^ CREAT,$/;" e enum:__anon1 +Calloc ./src/macros.h 23;" d +Cleanup ./src/macros.h 4;" d +Cleanup_unless ./src/macros.h 5;" d +Clone ./src/macros.h 9;" d +Cmapshared ./src/macros.h 36;" d +DEFAULTS_H ./src/defaults.h 2;" d +Debug ./src/macros.h 51;" d +ERROR ./src/defaults.h /^ ERROR, \/**< An error happened :-( *\/$/;" e enum:status_e_ +Eq ./src/macros.h 10;" d +Errno ./src/macros.h 68;" d +Errno_if ./src/macros.h 77;" d +Error ./src/macros.h 58;" d +Error_if ./src/macros.h 66;" d +F ./src/generate/gtask.h /^ int F; \/**< Arguments for fcntl syscall *\/$/;" m struct:gtask_s_ +FCHMOD ./src/opcodes.h /^ FCHMOD,$/;" e enum:__anon1 +FCHMODAT ./src/opcodes.h /^ FCHMODAT,$/;" e enum:__anon1 +FCHOWN ./src/opcodes.h /^ FCHOWN,$/;" e enum:__anon1 +FCHOWN16 ./src/opcodes.h /^ FCHOWN16,$/;" e enum:__anon1 +FCHOWNAT ./src/opcodes.h /^ FCHOWNAT,$/;" e enum:__anon1 +FCNTL ./src/opcodes.h /^ FCNTL = 70,$/;" e enum:__anon1 +FDATASYNC ./src/opcodes.h /^ FDATASYNC,$/;" e enum:__anon1 +FSTAT ./src/opcodes.h /^ FSTAT = 0,$/;" e enum:__anon1 +FSTATFS ./src/opcodes.h /^ FSTATFS,$/;" e enum:__anon1 +FSTATFS64 ./src/opcodes.h /^ FSTATFS64,$/;" e enum:__anon1 +FSTAT_AT ./src/opcodes.h /^ FSTAT_AT,$/;" e enum:__anon1 +FSYNC ./src/opcodes.h /^ FSYNC = 60,$/;" e enum:__anon1 +FUTILS_H ./src/utils/futils.h 2;" d +Fill_with_stuff ./src/macros.h 98;" d +Fopen ./src/macros.h 29;" d +G ./src/generate/gtask.h /^ int G; \/**< Arguments for fcntl syscall *\/$/;" m struct:gtask_s_ +GENERATE_H ./src/generate/generate.h 2;" d +GETDENTS ./src/opcodes.h /^ GETDENTS,$/;" e enum:__anon1 +GIOOP_H ./src/generate/gioop.h 2;" d +GPARSER_H ./src/generate/gparser.h 2;" d +GPROCESS_H ./src/generate/gprocess.h 2;" d +GTASK_H ./src/generate/gtask.h 2;" d +GWRITER_H ./src/generate/gwriter.h 2;" d +Gioop_write ./src/generate/gioop.h 12;" d +HMAP_H ./src/datas/hmap.h 2;" d +Has ./src/macros.h 17;" d +Hasnt ./src/macros.h 18;" d +INIT_H ./src/init/init.h 2;" d +IOREPLAY_COPYRIGHT ./src/defaults.h 19;" d +IOREPLAY_VERSION ./src/defaults.h 17;" d +ITASK_H ./src/init/itask.h 2;" d +ITHREAD_H ./src/init/ithread.h 2;" d +LCHOWN ./src/opcodes.h /^ LCHOWN,$/;" e enum:__anon1 +LCOWN16 ./src/opcodes.h /^ LCOWN16,$/;" e enum:__anon1 +LIST_H ./src/datas/list.h 2;" d +LSEEK ./src/opcodes.h /^ LSEEK,$/;" e enum:__anon1 +LSTAT ./src/opcodes.h /^ LSTAT,$/;" e enum:__anon1 +MACROS_H ./src/macros.h 2;" d +MAX_LINE_LEN ./src/defaults.h 13;" d +MAX_MOUNTPOINTS ./src/mounts.h 8;" d +MAX_TOKENS ./src/defaults.h 11;" d +META_EXIT ./src/opcodes.h /^ META_EXIT = 900,$/;" e enum:__anon1 +META_EXIT_GROUP ./src/opcodes.h /^ META_EXIT_GROUP,$/;" e enum:__anon1 +META_H ./src/meta/meta.h 2;" d +META_TIMELINE ./src/opcodes.h /^ META_TIMELINE$/;" e enum:__anon1 +MKDIR ./src/opcodes.h /^ MKDIR,$/;" e enum:__anon1 +MKDIR_AT ./src/opcodes.h /^ MKDIR_AT,$/;" e enum:__anon1 +MMAP2 ./src/opcodes.h /^ MMAP2 = 80,$/;" e enum:__anon1 +MOUNTPOINTS_H ./src/mounts.h 2;" d +MSYNC ./src/opcodes.h /^ MSYNC,$/;" e enum:__anon1 +MUNMAP ./src/opcodes.h /^ MUNMAP,$/;" e enum:__anon1 +Malloc ./src/macros.h 21;" d +Mmapshared ./src/macros.h 32;" d +Mset ./src/macros.h 25;" d +NAME_TO_HANDLE_AT ./src/opcodes.h /^ NAME_TO_HANDLE_AT,$/;" e enum:__anon1 +OPCODES_H ./src/opcodes.h 2;" d +OPEN ./src/opcodes.h /^ OPEN = 30,$/;" e enum:__anon1 +OPEN_AT ./src/opcodes.h /^ OPEN_AT,$/;" e enum:__anon1 +OPEN_BY_HANDLE_AT ./src/opcodes.h /^ OPEN_BY_HANDLE_AT,$/;" e enum:__anon1 +OPTIONS_H ./src/options.h 2;" d +Out ./src/macros.h 42;" d +Put ./src/macros.h 45;" d +RBUFFER_H ./src/datas/rbuffer.h 2;" d +READ ./src/opcodes.h /^ READ = 10,$/;" e enum:__anon1 +READAHEAD ./src/opcodes.h /^ READAHEAD,$/;" e enum:__anon1 +READDIR ./src/opcodes.h /^ READDIR,$/;" e enum:__anon1 +READLINK ./src/opcodes.h /^ READLINK,$/;" e enum:__anon1 +READLINK_AT ./src/opcodes.h /^ READLINK_AT,$/;" e enum:__anon1 +READV ./src/opcodes.h /^ READV,$/;" e enum:__anon1 +REMAP ./src/opcodes.h /^ REMAP,$/;" e enum:__anon1 +RENAME ./src/opcodes.h /^ RENAME = 40,$/;" e enum:__anon1 +RENAME_AT ./src/opcodes.h /^ RENAME_AT,$/;" e enum:__anon1 +RENAME_AT2 ./src/opcodes.h /^ RENAME_AT2,$/;" e enum:__anon1 +REPLAY_H ./src/replay/replay.h 2;" d +REPLAY_VERSION ./src/defaults.h 9;" d +RIOOP_H ./src/replay/rioop.h 2;" d +RMDIR ./src/opcodes.h /^ RMDIR,$/;" e enum:__anon1 +RPROCESS_H ./src/replay/rprocess.h 2;" d +RTASK_H ./src/replay/rtask.h 2;" d +RTHREAD_H ./src/replay/rthread.h 9;" d +RWORKER_H ./src/replay/rworker.h 2;" d +Readhex ./src/macros.h 14;" d +STACK_H ./src/datas/stack.h 2;" d +STAT ./src/opcodes.h /^ STAT,$/;" e enum:__anon1 +STATFS ./src/opcodes.h /^ STATFS,$/;" e enum:__anon1 +STATFS64 ./src/opcodes.h /^ STATFS64,$/;" e enum:__anon1 +SUCCESS ./src/defaults.h /^ SUCCESS, \/**< Great success! *\/$/;" e enum:status_e_ +SYNC ./src/opcodes.h /^ SYNC,$/;" e enum:__anon1 +SYNCFS ./src/opcodes.h /^ SYNCFS,$/;" e enum:__anon1 +SYNC_FILE_RANGE ./src/opcodes.h /^ SYNC_FILE_RANGE,$/;" e enum:__anon1 +Segfault ./src/macros.h 79;" d +TASK_BUFFER_PER_THREAD ./src/defaults.h 15;" d +UNKNOWN ./src/defaults.h /^ UNKNOWN, \/**< Unknown return status :-\/ *\/$/;" e enum:status_e_ +UNLINK ./src/opcodes.h /^ UNLINK,$/;" e enum:__anon1 +UNLINK_AT ./src/opcodes.h /^ UNLINK_AT,$/;" e enum:__anon1 +UTESTS_H ./src/utests.h 2;" d +UTILS_H ./src/utils/utils.h 2;" d +VFD_H ./src/vfd.h 2;" d +VSIZE_H ./src/generate/vsize.h 2;" d +WRITE ./src/opcodes.h /^ WRITE = 20,$/;" e enum:__anon1 +WRITEV ./src/opcodes.h /^ WRITEV,$/;" e enum:__anon1 +Warn ./src/macros.h 88;" d +Warn_if ./src/macros.h 95;" d +_Errno ./src/replay/rioop.c 15;" d file: +_Error ./src/replay/rioop.c 7;" d file: +_GNU_SOURCE ./src/utils/utils.h 5;" d +_Init_arg ./src/replay/rioop.c 24;" d file: +_Init_bytes ./src/replay/rioop.c 36;" d file: +_Init_cmd ./src/replay/rioop.c 25;" d file: +_Init_fd ./src/replay/rioop.c 26;" d file: +_Init_flags ./src/replay/rioop.c 27;" d file: +_Init_offset ./src/replay/rioop.c 29;" d file: +_Init_op ./src/replay/rioop.c 30;" d file: +_Init_path ./src/replay/rioop.c 32;" d file: +_Init_path2 ./src/replay/rioop.c 31;" d file: +_Init_rc ./src/replay/rioop.c 33;" d file: +_Init_virtfd ./src/replay/rioop.c 40;" d file: +_Init_whence ./src/replay/rioop.c 34;" d file: +_MAX_META_LEN ./src/meta/meta.c 3;" d file: +_MAX_PROCESSES ./src/generate/generate.c 10;" d file: +_PATH_INSERT ./src/mounts.c 5;" d file: +_PATH_INSERT_LEN ./src/mounts.c 6;" d file: +_Perc_filtered ./src/generate/generate.c 12;" d file: +_Set_dir ./src/generate/vsize.c 8;" d file: +_Set_file ./src/generate/vsize.c 7;" d file: +_Set_inserted ./src/generate/vsize.c 10;" d file: +_Set_renamed ./src/generate/vsize.c 11;" d file: +_Set_required ./src/generate/vsize.c 12;" d file: +_Set_unsure ./src/generate/vsize.c 9;" d file: +_Using_string_keys ./src/datas/hmap.c 3;" d file: +_amap_new ./src/datas/amap.c /^static amap_s *_amap_new(long size, bool mmapped)$/;" f file: +_amap_test ./src/datas/amap.c /^void _amap_test(amap_s *a)$/;" f +_arch_check_atomic ./src/main.c /^static void _arch_check_atomic(void)$/;" f file: +_gprocess_vfd_map_destroy_cb ./src/generate/gprocess.c /^void _gprocess_vfd_map_destroy_cb(void *data)$/;" f +_hmap_new ./src/datas/hmap.c /^hmap_s *_hmap_new(unsigned int init_size)$/;" f +_hmap_test ./src/datas/hmap.c /^static void _hmap_test(hmap_s *h)$/;" f file: +_hmap_test_l ./src/datas/hmap.c /^static void _hmap_test_l(hmap_s *h)$/;" f file: +_list_elem_remove ./src/datas/list.c /^void _list_elem_remove(list_s *l, list_elem_s *e)$/;" f +_print_help ./src/main.c /^static void _print_help(void)$/;" f file: +_print_synopsis ./src/main.c /^static void _print_synopsis(void)$/;" f file: +_print_version ./src/main.c /^static void _print_version(void)$/;" f file: +_rprocess_destroy_cb ./src/replay/rworker.c /^static void _rprocess_destroy_cb(void *data)$/;" f file: +_rthread_init_log ./src/replay/rthread.c /^static void _rthread_init_log(rthread_s *t)$/;" f file: +absolute_path ./src/utils/futils.c /^char *absolute_path(const char *path)$/;" f +address ./src/generate/gtask.h /^ long address; \/**< An address (used by mmap related syscalls) *\/$/;" m struct:gtask_s_ +address2 ./src/generate/gtask.h /^ long address2; \/**< Another address (used by mmap related syscalls) *\/$/;" m struct:gtask_s_ +amap_destroy ./src/datas/amap.c /^void amap_destroy(amap_s* a)$/;" f +amap_get ./src/datas/amap.c /^void* amap_get(amap_s *a, const long position)$/;" f +amap_new ./src/datas/amap.c /^amap_s* amap_new(const long size)$/;" f +amap_new_mmapped ./src/datas/amap.c /^amap_s* amap_new_mmapped(const long size)$/;" f +amap_print ./src/datas/amap.c /^void amap_print(amap_s* a)$/;" f +amap_reset ./src/datas/amap.c /^void amap_reset(amap_s* a)$/;" f +amap_run_cb ./src/datas/amap.c /^void amap_run_cb(amap_s *a, void (*cb)(void *data))$/;" f +amap_s ./src/datas/amap.h /^} amap_s;$/;" t typeref:struct:amap_s_ +amap_s_ ./src/datas/amap.h /^typedef struct amap_s_ {$/;" s +amap_set ./src/datas/amap.c /^int amap_set(amap_s *a, const long position, void* value)$/;" f +amap_test ./src/datas/amap.c /^void amap_test(void)$/;" f +amap_unset ./src/datas/amap.c /^void* amap_unset(amap_s *a, const long position)$/;" f +append_random_to_file ./src/utils/futils.c /^void append_random_to_file(char *path, unsigned long bytes)$/;" f +arrays ./src/datas/amap.h /^ void*** arrays; \/**< The pointers to the amap arrays *\/$/;" m struct:amap_s_ +btree_destroy ./src/datas/btree.c /^void btree_destroy(btree_s* b)$/;" f +btree_destroy2 ./src/datas/btree.c /^void btree_destroy2(btree_s* b)$/;" f +btree_get ./src/datas/btree.c /^void* btree_get(btree_s* b, int key)$/;" f +btree_insert ./src/datas/btree.c /^int btree_insert(btree_s* b, int key, void *data)$/;" f +btree_new ./src/datas/btree.c /^btree_s* btree_new()$/;" f +btree_print ./src/datas/btree.c /^void btree_print(btree_s* b)$/;" f +btree_s ./src/datas/btree.h /^} btree_s;$/;" t typeref:struct:btree_s_ +btree_s_ ./src/datas/btree.h /^typedef struct btree_s_ {$/;" s +btreelem_ ./src/datas/btree.h /^typedef struct btreelem_ {$/;" s +btreelem_destroy_r ./src/datas/btree.c /^void btreelem_destroy_r(btreelem_s* e)$/;" f +btreelem_destroy_r2 ./src/datas/btree.c /^void btreelem_destroy_r2(btreelem_s* e)$/;" f +btreelem_get_r ./src/datas/btree.c /^void* btreelem_get_r(btreelem_s* e, int key)$/;" f +btreelem_insert_r ./src/datas/btree.c /^int btreelem_insert_r(btreelem_s* e, int key, void *data)$/;" f +btreelem_new ./src/datas/btree.c /^btreelem_s* btreelem_new(int key, void *data)$/;" f +btreelem_print_r ./src/datas/btree.c /^void btreelem_print_r(btreelem_s* e, int depth)$/;" f +btreelem_s ./src/datas/btree.h /^} btreelem_s;$/;" t typeref:struct:btreelem_ +bytes ./src/generate/gtask.h /^ long bytes; \/**< Amount of bytes *\/$/;" m struct:gtask_s_ +cache_file ./src/utils/futils.c /^void cache_file(const char *file)$/;" f +capture_file ./src/options.h /^ char *capture_file; \/**< The name of the .capture file *\/$/;" m struct:options_s_ +chown_path ./src/utils/futils.c /^void chown_path(const char *user, const char *path)$/;" f +chreplace ./src/utils/utils.c /^void chreplace(char *str, char replace, char with)$/;" f +cleanup_run ./src/cleanup/cleanup.c /^status_e cleanup_run(options_s *opts)$/;" f +clone ./src/replay/rtask.h /^ char *clone; \/**< Used for debug purposes only *\/$/;" m struct:rtask_s_ +count ./src/generate/gtask.h /^ long count; \/**< A count *\/$/;" m struct:gtask_s_ +count ./src/mounts.h /^ int count; \/**< The amount of mount points *\/$/;" m struct:mounts_s_ +data ./src/datas/btree.h /^ void *data; \/**< A pointer to the data stored in this element *\/$/;" m struct:btreelem_ +data ./src/datas/hmap.h /^ void **data; \/**< Pointers to the stored data, NULL if nothing there *\/$/;" m struct:hmap_s_ +data ./src/datas/list.h /^ void *data; \/**< Pointer to the stored data *\/$/;" m struct:list_elem_s_ +data ./src/datas/stack.h /^ void *data; \/**< Pointer to the stored data in the current element *\/$/;" m struct:stack_elem_s_ +data_destroy ./src/datas/amap.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all elements *\/$/;" m struct:amap_s_ +data_destroy ./src/datas/hmap.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all data *\/$/;" m struct:hmap_s_ +data_destroy ./src/datas/list.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all data *\/$/;" m struct:list_s_ +debug ./src/vfd.h /^ int debug; \/**< Used for debugging purposes only *\/$/;" m struct:vfd_s_ +dirfd ./src/vfd.h /^ DIR *dirfd; \/**< The real dirfd *\/$/;" m struct:vfd_s_ +dirname_r ./src/utils/futils.c /^char* dirname_r(char *path)$/;" f +dirs_created ./src/init/itask.h /^ long dirs_created;$/;" m struct:itask_s_ +drop_caches ./src/options.h /^ bool drop_caches; \/**< True if ioreplay should drop all Linux caches *\/$/;" m struct:options_s_ +drop_caches ./src/utils/futils.c /^void drop_caches(void)$/;" f +drop_root ./src/utils/utils.c /^void drop_root(const char *user)$/;" f +ensure_dir_empty ./src/utils/futils.c /^void ensure_dir_empty(const char *path)$/;" f +ensure_dir_exists ./src/utils/futils.c /^long ensure_dir_exists(const char *path)$/;" f +ensure_file_exists ./src/utils/futils.c /^int ensure_file_exists(char *path, long *num_dirs_created)$/;" f +ensure_parent_dir_exists ./src/utils/futils.c /^void ensure_parent_dir_exists(const char *path)$/;" f +exists ./src/utils/futils.c /^bool exists(const char *path)$/;" f +fd ./src/generate/gtask.h /^ int fd; \/**< File descriptor number *\/$/;" m struct:gtask_s_ +fd ./src/vfd.h /^ int fd; \/**< the real fd *\/$/;" m struct:vfd_s_ +fd_map ./src/generate/gprocess.h /^ hmap_s *fd_map; \/**< All mappings from real fd to virtual fd *\/$/;" m struct:gprocess_s_ +fds_map ./src/replay/rprocess.h /^ amap_s *fds_map; \/**< Holding all file descriptors *\/$/;" m struct:rprocess_s_ +fds_map ./src/replay/rworker.h /^ amap_s* fds_map; \/**< Holding all file descriptors *\/$/;" m struct:__anon2 +files_created ./src/init/itask.h /^ long files_created;$/;" m struct:itask_s_ +filtered_where ./src/generate/gtask.h /^ char *filtered_where; \/**< Only used for debugging purposes *\/$/;" m struct:gtask_s_ +first ./src/datas/list.h /^ list_elem_s *first; \/**< The first element, NULL if list empty *\/$/;" m struct:list_s_ +flags ./src/generate/gtask.h /^ int flags; \/**< File open flags *\/$/;" m struct:gtask_s_ +fnotnull ./src/utils/utils.c /^FILE* fnotnull(FILE *fd, const char *path, char *file, int line)$/;" f +free_path ./src/vfd.h /^ bool free_path; \/**< True if path has to be freed or not *\/$/;" m struct:vfd_s_ +generate ./src/generate/gparser.h /^ generate_s *generate; \/**< The generate object *\/$/;" m struct:gparser_s_ +generate ./src/generate/gtask.h /^ void *generate; \/**< A pointer to the generate object *\/$/;" m struct:gtask_s_ +generate ./src/generate/gwriter.h /^ struct generate_s_ *generate; \/**< The generate object *\/$/;" m struct:gwriter_s_ typeref:struct:gwriter_s_::generate_s_ +generate ./src/generate/vsize.h /^ void *generate; \/**< A pointer to the generate object *\/$/;" m struct:vsize_s_ +generate_destroy ./src/generate/generate.c /^void generate_destroy(generate_s *g)$/;" f +generate_gprocess_by_realpid ./src/generate/generate.c /^void generate_gprocess_by_realpid(generate_s *g, gtask_s *t)$/;" f +generate_new ./src/generate/generate.c /^generate_s* generate_new(options_s *opts)$/;" f +generate_run ./src/generate/generate.c /^status_e generate_run(options_s *opts)$/;" f +generate_s ./src/generate/generate.h /^} generate_s;$/;" t typeref:struct:generate_s_ +generate_s_ ./src/generate/generate.h /^typedef struct generate_s_ {$/;" s +generate_vsize_by_path ./src/generate/generate.c /^vsize_s* generate_vsize_by_path(generate_s *g, gtask_s *t,$/;" f +generate_write_init_cb ./src/generate/generate.c /^void generate_write_init_cb(void *data)$/;" f +get_loadavg ./src/utils/utils.c /^void get_loadavg(char *readbuf)$/;" f +gioop_chmod ./src/generate/gioop.c /^status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_chown ./src/generate/gioop.c /^status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_close ./src/generate/gioop.c /^status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_close_all_vfd_cb ./src/generate/gioop.c /^void gioop_close_all_vfd_cb(void *data, void *data2)$/;" f +gioop_creat ./src/generate/gioop.c /^status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_exit_group ./src/generate/gioop.c /^status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fchmod ./src/generate/gioop.c /^status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fchown ./src/generate/gioop.c /^status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fcntl ./src/generate/gioop.c /^status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fdatasync ./src/generate/gioop.c /^status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstat ./src/generate/gioop.c /^status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstatat ./src/generate/gioop.c /^status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstatfs ./src/generate/gioop.c /^status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstatfs64 ./src/generate/gioop.c /^status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fsync ./src/generate/gioop.c /^status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_getdents ./src/generate/gioop.c /^status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_lchown ./src/generate/gioop.c /^status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_lseek ./src/generate/gioop.c /^status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_lstat ./src/generate/gioop.c /^status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_mkdir ./src/generate/gioop.c /^status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_mkdirat ./src/generate/gioop.c /^status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_open ./src/generate/gioop.c /^status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_openat ./src/generate/gioop.c /^status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_read ./src/generate/gioop.c /^status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readahead ./src/generate/gioop.c /^status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readdir ./src/generate/gioop.c /^status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readlink ./src/generate/gioop.c /^status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readlinkat ./src/generate/gioop.c /^status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readv ./src/generate/gioop.c /^status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_rename ./src/generate/gioop.c /^status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_renameat ./src/generate/gioop.c /^status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_renameat2 ./src/generate/gioop.c /^status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_rmdir ./src/generate/gioop.c /^status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_run ./src/generate/gioop.c /^status_e gioop_run(gwriter_s *w, gtask_s *t)$/;" f +gioop_stat ./src/generate/gioop.c /^status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_statfs ./src/generate/gioop.c /^status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_statfs64 ./src/generate/gioop.c /^status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_sync ./src/generate/gioop.c /^status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_sync_file_range ./src/generate/gioop.c /^status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_syncfs ./src/generate/gioop.c /^status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_unlink ./src/generate/gioop.c /^status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_unlinkat ./src/generate/gioop.c /^status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_write ./src/generate/gioop.c /^status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_writev ./src/generate/gioop.c /^status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gparser_destroy ./src/generate/gparser.c /^void gparser_destroy(gparser_s *p)$/;" f +gparser_extract ./src/generate/gparser.c /^void gparser_extract(gparser_s *p, gtask_s *t)$/;" f +gparser_extract_tok ./src/generate/gparser.c /^status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok)$/;" f +gparser_get_pidtid ./src/generate/gparser.c /^bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid)$/;" f +gparser_new ./src/generate/gparser.c /^gparser_s* gparser_new(generate_s *g)$/;" f +gparser_pthread_start ./src/generate/gparser.c /^void* gparser_pthread_start(void *data)$/;" f +gparser_s ./src/generate/gparser.h /^} gparser_s;$/;" t typeref:struct:gparser_s_ +gparser_s_ ./src/generate/gparser.h /^typedef struct gparser_s_ {$/;" s +gparser_start ./src/generate/gparser.c /^void gparser_start(gparser_s *p)$/;" f +gparser_terminate ./src/generate/gparser.c /^void gparser_terminate(gparser_s *p)$/;" f +gparser_token_not_ok ./src/generate/gparser.c /^bool gparser_token_not_ok(gparser_s *p, char *tok)$/;" f +gprocess ./src/generate/gtask.h /^ struct gprocess_s_ *gprocess; \/**< A pointer to the process object *\/$/;" m struct:gtask_s_ typeref:struct:gtask_s_::gprocess_s_ +gprocess_create_vfd_by_realfd ./src/generate/gprocess.c /^void gprocess_create_vfd_by_realfd(gprocess_s *gp, gtask_s *t, generate_s *g)$/;" f +gprocess_destroy ./src/generate/gprocess.c /^void gprocess_destroy(gprocess_s *gp)$/;" f +gprocess_new ./src/generate/gprocess.c /^gprocess_s* gprocess_new(const long pid, const long mapped_pid)$/;" f +gprocess_s ./src/generate/gprocess.h /^} gprocess_s;$/;" t typeref:struct:gprocess_s_ +gprocess_s_ ./src/generate/gprocess.h /^typedef struct gprocess_s_ {$/;" s +gprocess_vfd_by_realfd ./src/generate/gprocess.c /^status_e gprocess_vfd_by_realfd(gprocess_s *gp, gtask_s *t)$/;" f +gtask_destroy ./src/generate/gtask.c /^void gtask_destroy(gtask_s *t)$/;" f +gtask_init ./src/generate/gtask.c /^void gtask_init(gtask_s *t, char *line, const unsigned long lineno)$/;" f +gtask_new ./src/generate/gtask.c /^gtask_s* gtask_new(void *generate)$/;" f +gtask_s ./src/generate/gtask.h /^} gtask_s;$/;" t typeref:struct:gtask_s_ +gtask_s_ ./src/generate/gtask.h /^typedef struct gtask_s_ {$/;" s +gwriter_destroy ./src/generate/gwriter.c /^void gwriter_destroy(gwriter_s *w)$/;" f +gwriter_new ./src/generate/gwriter.c /^gwriter_s* gwriter_new(generate_s *g)$/;" f +gwriter_pthread_start ./src/generate/gwriter.c /^void* gwriter_pthread_start(void *data)$/;" f +gwriter_s ./src/generate/gwriter.h /^} gwriter_s;$/;" t typeref:struct:gwriter_s_ +gwriter_s_ ./src/generate/gwriter.h /^typedef struct gwriter_s_ {$/;" s +gwriter_start ./src/generate/gwriter.c /^void gwriter_start(gwriter_s *w)$/;" f +gwriter_terminate ./src/generate/gwriter.c /^void gwriter_terminate(gwriter_s *w)$/;" f +has_fd ./src/generate/gtask.h /^ bool has_fd; \/**< True if task has a file descriptor number *\/$/;" m struct:gtask_s_ +hmap_destroy ./src/datas/hmap.c /^void hmap_destroy(hmap_s *h)$/;" f +hmap_get ./src/datas/hmap.c /^void* hmap_get(hmap_s *h, char *key)$/;" f +hmap_get_addr ./src/datas/hmap.c /^unsigned int hmap_get_addr(hmap_s *h, char *key)$/;" f +hmap_get_addr_l ./src/datas/hmap.c /^unsigned int hmap_get_addr_l(hmap_s *h, const long key)$/;" f +hmap_get_l ./src/datas/hmap.c /^void* hmap_get_l(hmap_s *h, const long key)$/;" f +hmap_insert ./src/datas/hmap.c /^int hmap_insert(hmap_s *h, char *key, void *data)$/;" f +hmap_insert_l ./src/datas/hmap.c /^int hmap_insert_l(hmap_s *h, const long key, void *data)$/;" f +hmap_new ./src/datas/hmap.c /^hmap_s *hmap_new(unsigned int init_size)$/;" f +hmap_new_l ./src/datas/hmap.c /^hmap_s *hmap_new_l(unsigned int init_size)$/;" f +hmap_print ./src/datas/hmap.c /^void hmap_print(hmap_s *h)$/;" f +hmap_remove ./src/datas/hmap.c /^void* hmap_remove(hmap_s *h, char *key)$/;" f +hmap_remove_l ./src/datas/hmap.c /^void* hmap_remove_l(hmap_s *h, const long key)$/;" f +hmap_run_cb ./src/datas/hmap.c /^void hmap_run_cb(hmap_s* h, void (*cb)(void *data))$/;" f +hmap_run_cb2 ./src/datas/hmap.c /^void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_)$/;" f +hmap_s ./src/datas/hmap.h /^} hmap_s;$/;" t typeref:struct:hmap_s_ +hmap_s_ ./src/datas/hmap.h /^typedef struct hmap_s_ {$/;" s +hmap_test ./src/datas/hmap.c /^void hmap_test(void)$/;" f +id ./src/generate/vsize.h /^ unsigned long id; \/**< The vsize id *\/$/;" m struct:vsize_s_ +ignore_count ./src/mounts.h /^ int ignore_count; \/**< The amount of ignored mount points *\/$/;" m struct:mounts_s_ +ignore_mps ./src/mounts.h /^ char *ignore_mps[MAX_MOUNTPOINTS]; \/**< The ignored mp paths *\/$/;" m struct:mounts_s_ +init ./src/init/ithread.h /^ init_s *init; \/**< The responsible init object *\/$/;" m struct:ithread_s_ +init ./src/options.h /^ bool init; \/**< If set ioreplay will initialise the environment *\/$/;" m struct:options_s_ +init_destroy ./src/init/init.c /^void init_destroy(init_s *i)$/;" f +init_extract_header ./src/init/init.c /^void init_extract_header(init_s *i, off_t *init_offset)$/;" f +init_new ./src/init/init.c /^init_s *init_new(options_s *opts)$/;" f +init_parent_dir ./src/generate/vsize.c /^void init_parent_dir(vsize_s *v, const char *path)$/;" f +init_run ./src/init/init.c /^status_e init_run(options_s *opts)$/;" f +init_s ./src/init/init.h /^} init_s;$/;" t typeref:struct:init_s_ +init_s_ ./src/init/init.h /^typedef struct init_s_ {$/;" s +initm ./src/replay/rprocess.h /^ bool initm; \/**< Indicates whether ioreplay is in init mode or not *\/$/;" m struct:rprocess_s_ +inserted ./src/generate/vsize.h /^ bool inserted; \/**< For debugging purposes only *\/$/;" m struct:vsize_s_ +is_dir ./src/generate/vsize.h /^ bool is_dir; \/**< True if this file\/dir is a directory *\/$/;" m struct:vsize_s_ +is_dir ./src/init/itask.h /^ bool is_dir;$/;" m struct:itask_s_ +is_dir ./src/utils/futils.c /^bool is_dir(const char *path)$/;" f +is_file ./src/generate/vsize.h /^ bool is_file; \/**< True if this file\/dir is a regular file *\/$/;" m struct:vsize_s_ +is_file ./src/init/itask.h /^ bool is_file;$/;" m struct:itask_s_ +is_number ./src/utils/utils.c /^bool is_number(char *str)$/;" f +is_reg ./src/utils/futils.c /^bool is_reg(const char *path)$/;" f +itask_destroy ./src/init/itask.c /^void itask_destroy(itask_s *task)$/;" f +itask_extract_stats ./src/init/itask.c /^void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created,$/;" f +itask_new ./src/init/itask.c /^itask_s* itask_new()$/;" f +itask_print ./src/init/itask.c /^void itask_print(itask_s *task)$/;" f +itask_reset_stats ./src/init/itask.c /^void itask_reset_stats(itask_s *task)$/;" f +itask_s ./src/init/itask.h /^} itask_s;$/;" t typeref:struct:itask_s_ +itask_s_ ./src/init/itask.h /^typedef struct itask_s_ {$/;" s +ithread_destroy ./src/init/ithread.c /^void ithread_destroy(ithread_s *t)$/;" f +ithread_new ./src/init/ithread.c /^ithread_s* ithread_new(init_s *i)$/;" f +ithread_pthread_start ./src/init/ithread.c /^void* ithread_pthread_start(void *data)$/;" f +ithread_run_task ./src/init/ithread.c /^void ithread_run_task(ithread_s *t, itask_s *task)$/;" f +ithread_s ./src/init/ithread.h /^} ithread_s;$/;" t typeref:struct:ithread_s_ +ithread_s_ ./src/init/ithread.h /^typedef struct ithread_s_ {$/;" s +ithread_start ./src/init/ithread.c /^void ithread_start(ithread_s *t)$/;" f +ithread_terminate ./src/init/ithread.c /^void ithread_terminate(ithread_s *t)$/;" f +key ./src/datas/btree.h /^ int key; \/**< The key of the element *\/$/;" m struct:btreelem_ +key ./src/datas/list.h /^ char *key; \/**< The key of the lemenet *\/$/;" m struct:list_elem_s_ +key_l ./src/datas/list.h /^ long key_l; \/**< The same as key, but for long keys *\/$/;" m struct:list_elem_s_ +keys ./src/datas/hmap.h /^ char **keys; \/**< List of all keys, NULL if nothing at a address *\/$/;" m struct:hmap_s_ +keys_l ./src/datas/hmap.h /^ int *keys_l; \/**< Same as keys, but for long keys *\/$/;" m struct:hmap_s_ +l ./src/datas/hmap.h /^ list_s **l; \/**< Pointers to the linked lists, used on hash collision *\/$/;" m struct:hmap_s_ +left ./src/datas/btree.h /^ struct btreelem_ *left; \/**< The next element to the left *\/$/;" m struct:btreelem_ typeref:struct:btreelem_::btreelem_ +lengths ./src/mounts.h /^ int lengths[MAX_MOUNTPOINTS]; \/**< The mp lenghts *\/$/;" m struct:mounts_s_ +line ./src/generate/gtask.h /^ char *line; \/**< A pointer to the remaining part of the .capture line *\/$/;" m struct:gtask_s_ +line ./src/replay/rtask.h /^ char line[MAX_LINE_LEN]; \/**< The remaining part of the .replay line *\/$/;" m struct:rtask_s_ +lineno ./src/generate/generate.h /^ long lineno; \/**< The current line number *\/$/;" m struct:generate_s_ +lineno ./src/generate/gtask.h /^ long lineno; \/**< The current line number *\/$/;" m struct:gtask_s_ +lineno ./src/replay/rprocess.h /^ unsigned long lineno; \/**< Holding the current .replay line number *\/$/;" m struct:rprocess_s_ +lineno ./src/replay/rtask.h /^ unsigned long lineno; \/**< The current line number *\/$/;" m struct:rtask_s_ +list_destroy ./src/datas/list.c /^void list_destroy(list_s *l)$/;" f +list_elem_s ./src/datas/list.h /^} list_elem_s;$/;" t typeref:struct:list_elem_s_ +list_elem_s_ ./src/datas/list.h /^typedef struct list_elem_s_ {$/;" s +list_key_get ./src/datas/list.c /^void* list_key_get(list_s *l, char *key)$/;" f +list_key_get_l ./src/datas/list.c /^void* list_key_get_l(list_s *l, const long key)$/;" f +list_key_insert ./src/datas/list.c /^int list_key_insert(list_s *l, char *key, void *data)$/;" f +list_key_insert_l ./src/datas/list.c /^int list_key_insert_l(list_s *l, const long key, void *data)$/;" f +list_key_remove ./src/datas/list.c /^void* list_key_remove(list_s *l, char *key)$/;" f +list_key_remove_l ./src/datas/list.c /^void* list_key_remove_l(list_s *l, const long key)$/;" f +list_new ./src/datas/list.c /^list_s *list_new()$/;" f +list_new_l ./src/datas/list.c /^list_s *list_new_l()$/;" f +list_print ./src/datas/list.c /^void list_print(list_s *l)$/;" f +list_run_cb ./src/datas/list.c /^void list_run_cb(list_s* l, void (*cb)(void *data))$/;" f +list_run_cb2 ./src/datas/list.c /^void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_)$/;" f +list_s ./src/datas/list.h /^} list_s;$/;" t typeref:struct:list_s_ +list_s_ ./src/datas/list.h /^typedef struct list_s_ {$/;" s +list_test ./src/datas/list.c /^void list_test(void)$/;" f +main ./src/main.c /^int main(int argc, char **argv)$/;" f +mapped_fd ./src/generate/gtask.h /^ long mapped_fd; \/**< The mapped file descriptor number *\/$/;" m struct:gtask_s_ +mapped_fd ./src/vfd.h /^ long mapped_fd; \/**< The mapped fd (virtual fd) *\/$/;" m struct:vfd_s_ +mapped_pid ./src/generate/gprocess.h /^ long mapped_pid; \/**< The mapped PID *\/$/;" m struct:gprocess_s_ +mapped_time ./src/generate/gtask.h /^ long mapped_time; \/**< The mapped time *\/$/;" m struct:gtask_s_ +max_mapped_fd ./src/generate/gprocess.h /^ long max_mapped_fd; \/**< The max mapped fd number *\/$/;" m struct:gprocess_s_ +meta_destroy ./src/meta/meta.c /^void meta_destroy(meta_s *m)$/;" f +meta_new ./src/meta/meta.c /^meta_s* meta_new(FILE *replay_fd)$/;" f +meta_read_l ./src/meta/meta.c /^bool meta_read_l(meta_s *m, char *key, long *val)$/;" f +meta_read_s ./src/meta/meta.c /^bool meta_read_s(meta_s *m, char *key, char **val)$/;" f +meta_read_start ./src/meta/meta.c /^void meta_read_start(meta_s *m)$/;" f +meta_reserve ./src/meta/meta.c /^void meta_reserve(meta_s *m)$/;" f +meta_s ./src/meta/meta.h /^} meta_s;$/;" t typeref:struct:meta_s_ +meta_s_ ./src/meta/meta.h /^typedef struct meta_s_ {$/;" s +meta_write_l ./src/meta/meta.c /^void meta_write_l(meta_s *m, char *key, long val)$/;" f +meta_write_s ./src/meta/meta.c /^void meta_write_s(meta_s *m, char *key, char *val)$/;" f +meta_write_start ./src/meta/meta.c /^void meta_write_start(meta_s *m)$/;" f +mkdir_p ./src/utils/futils.c /^int mkdir_p(const char *path, mode_t mode, long *num_dirs_created)$/;" f +mmap_map ./src/generate/generate.h /^ hmap_s *mmap_map; \/**< mmap address mappings *\/$/;" m struct:generate_s_ +mmapok ./src/utils/utils.c /^void* mmapok(void *p, char *file, int line)$/;" f +mmapped ./src/datas/amap.h /^ bool mmapped; \/**< True if amap is memory mapped *\/$/;" m struct:amap_s_ +mode ./src/generate/gtask.h /^ int mode; \/**< File open mode *\/$/;" m struct:gtask_s_ +mounts ./src/init/init.h /^ mounts_s *mounts;$/;" m struct:init_s_ +mounts_destroy ./src/mounts.c /^void mounts_destroy(mounts_s *m)$/;" f +mounts_get_mountnumber ./src/mounts.c /^int mounts_get_mountnumber(mounts_s *m, const char *path)$/;" f +mounts_ignore_path ./src/mounts.c /^bool mounts_ignore_path(mounts_s *m, const char *path)$/;" f +mounts_init ./src/mounts.c /^void mounts_init(mounts_s *m)$/;" f +mounts_new ./src/mounts.c /^mounts_s *mounts_new(options_s *opts)$/;" f +mounts_purge ./src/mounts.c /^void mounts_purge(mounts_s *m)$/;" f +mounts_read ./src/mounts.c /^void mounts_read(mounts_s *m)$/;" f +mounts_s ./src/mounts.h /^} mounts_s;$/;" t typeref:struct:mounts_s_ +mounts_s_ ./src/mounts.h /^typedef struct mounts_s_ {$/;" s +mounts_transform_path ./src/mounts.c /^bool mounts_transform_path(mounts_s *m, const char *name,$/;" f +mounts_trash ./src/mounts.c /^void mounts_trash(mounts_s *m)$/;" f +mps ./src/generate/generate.h /^ mounts_s *mps; \/**< The mounts object *\/$/;" m struct:generate_s_ +mps ./src/mounts.h /^ char *mps[MAX_MOUNTPOINTS]; \/**< The mp paths *\/$/;" m struct:mounts_s_ +name ./src/generate/generate.h /^ char *name; \/**< The name of the test specified by the user *\/$/;" m struct:generate_s_ +name ./src/options.h /^ char *name; \/**< The name of the test (found in .ioreplay\/name sub-dirs) *\/$/;" m struct:options_s_ +next ./src/datas/list.h /^ struct list_elem_s_ *next; \/**< The next element *\/$/;" m struct:list_elem_s_ typeref:struct:list_elem_s_::list_elem_s_ +next ./src/datas/stack.h /^ struct stack_elem_s_ *next; \/**< The next element *\/$/;" m struct:stack_elem_s_ typeref:struct:stack_elem_s_::stack_elem_s_ +notnull ./src/utils/utils.c /^void* notnull(void *p, char *file, int line, int count)$/;" f +num_arrays ./src/datas/amap.h /^ int num_arrays; \/**< The amount of arrays used in the amap *\/$/;" m struct:amap_s_ +num_lines_filtered ./src/generate/generate.h /^ long num_lines_filtered; \/**< The amount of lines filtered out *\/$/;" m struct:generate_s_ +num_mapped_fds ./src/generate/generate.h /^ unsigned long num_mapped_fds; \/**< The amount of mapped FDs *\/$/;" m struct:generate_s_ +num_mapped_pids ./src/generate/generate.h /^ unsigned long num_mapped_pids; \/**< The amount of mapped PIDs *\/$/;" m struct:generate_s_ +num_threads_per_worker ./src/options.h /^ int num_threads_per_worker; \/**< Max threads per worker processes *\/$/;" m struct:options_s_ +num_vsizes ./src/generate/generate.h /^ unsigned long num_vsizes; \/**< The amount of virtual sizes *\/$/;" m struct:generate_s_ +num_workers ./src/options.h /^ int num_workers; \/**< The amount of worker processes *\/$/;" m struct:options_s_ +offset ./src/generate/gtask.h /^ long offset; \/**< A offset *\/$/;" m struct:gtask_s_ +offset ./src/generate/vsize.h /^ off_t offset; \/**< The current file offset *\/$/;" m struct:vsize_s_ +offset ./src/meta/meta.h /^ off_t offset; \/**< The meta offset (usually 0) *\/$/;" m struct:meta_s_ +offset ./src/vfd.h /^ unsigned long offset; \/**< The current virtual file offset in bytes *\/$/;" m struct:vfd_s_ +op ./src/generate/gtask.h /^ char *op; \/**< Operation\/syscall name *\/$/;" m struct:gtask_s_ +opcode_e ./src/opcodes.h /^} opcode_e;$/;" t typeref:enum:__anon1 +options_destroy ./src/options.c /^void options_destroy(options_s *o)$/;" f +options_new ./src/options.c /^options_s *options_new()$/;" f +options_s ./src/options.h /^} options_s;$/;" t typeref:struct:options_s_ +options_s_ ./src/options.h /^typedef struct options_s_ {$/;" s +opts ./src/generate/generate.h /^ options_s *opts; \/**< A pointer to the options object *\/$/;" m struct:generate_s_ +opts ./src/init/init.h /^ options_s *opts;$/;" m struct:init_s_ +opts ./src/mounts.h /^ options_s *opts; \/**< A pointer to the options object *\/$/;" m struct:mounts_s_ +opts ./src/replay/rworker.h /^ options_s *opts; \/**< To synchronise access to rthread_buffer *\/$/;" m struct:__anon2 +original_line ./src/generate/gtask.h /^ char *original_line; \/**< Only used for debugging purposes *\/$/;" m struct:gtask_s_ +path ./src/generate/gtask.h /^ char *path; \/**< Path name *\/$/;" m struct:gtask_s_ +path ./src/generate/vsize.h /^ char *path; \/**< The path to the file\/directory *\/$/;" m struct:vsize_s_ +path ./src/init/itask.h /^ char *path;$/;" m struct:itask_s_ +path ./src/vfd.h /^ char *path; \/**< The file path belonging to that fd *\/$/;" m struct:vfd_s_ +path2 ./src/generate/gtask.h /^ char *path2; \/**< A second path name (e.g. for rename) *\/$/;" m struct:gtask_s_ +path2_r ./src/generate/gtask.h /^ char *path2_r; \/**< Work around to track mallocs, so it can be freed *\/$/;" m struct:gtask_s_ +path_r ./src/generate/gtask.h /^ char *path_r; \/**< Work around to track mallocs, so it can be freed *\/$/;" m struct:gtask_s_ +pid ./src/generate/gprocess.h /^ long pid; \/**< The real PID *\/$/;" m struct:gprocess_s_ +pid ./src/generate/gtask.h /^ long pid; \/**< The process ID *\/$/;" m struct:gtask_s_ +pid ./src/replay/rprocess.h /^ int pid; \/**< The virtual process ID *\/$/;" m struct:rprocess_s_ +pid_map ./src/generate/generate.h /^ amap_s *pid_map; \/**< A map of all virtual process objects *\/$/;" m struct:generate_s_ +pidtid ./src/generate/gtask.h /^ char *pidtid; \/**< String representing pid:tid *\/$/;" m struct:gtask_s_ +prev ./src/datas/list.h /^ struct list_elem_s_ *prev; \/**< The previous element *\/$/;" m struct:list_elem_s_ typeref:struct:list_elem_s_::list_elem_s_ +process ./src/replay/rtask.h /^ void *process; \/* The responsible process object *\/$/;" m struct:rtask_s_ +pthread ./src/generate/gparser.h /^ pthread_t pthread; \/**< The posix thread *\/$/;" m struct:gparser_s_ +pthread ./src/generate/gwriter.h /^ pthread_t pthread; \/**< The posix thread *\/$/;" m struct:gwriter_s_ +pthread ./src/init/ithread.h /^ pthread_t pthread; \/**< We run the init tasks in concurrent pthreads *\/$/;" m struct:ithread_s_ +pthread ./src/replay/rthread.h /^ pthread_t pthread; \/**< We run the tasks in concurrent pthreads *\/$/;" m struct:rthread_s_ +purge ./src/options.h /^ bool purge; \/**< If set ioreplay will purge the environment *\/$/;" m struct:options_s_ +queue ./src/generate/gparser.h /^ rbuffer_s *queue; \/**< A queue of task objects *\/$/;" m struct:gparser_s_ +queue ./src/generate/gwriter.h /^ rbuffer_s *queue; \/**< A queue of task objects *\/$/;" m struct:gwriter_s_ +queue ./src/init/ithread.h /^ rbuffer_s *queue; \/**< The thread's task queue *\/$/;" m struct:ithread_s_ +rbuffer_destroy ./src/datas/rbuffer.c /^void rbuffer_destroy(rbuffer_s *r)$/;" f +rbuffer_get_next ./src/datas/rbuffer.c /^void* rbuffer_get_next(rbuffer_s* r)$/;" f +rbuffer_has_next ./src/datas/rbuffer.c /^bool rbuffer_has_next(rbuffer_s* r)$/;" f +rbuffer_insert ./src/datas/rbuffer.c /^bool rbuffer_insert(rbuffer_s* r, void *data)$/;" f +rbuffer_new ./src/datas/rbuffer.c /^rbuffer_s *rbuffer_new(const int size)$/;" f +rbuffer_print ./src/datas/rbuffer.c /^void rbuffer_print(rbuffer_s* r)$/;" f +rbuffer_s ./src/datas/rbuffer.h /^} rbuffer_s;$/;" t typeref:struct:rbuffer_s_ +rbuffer_s_ ./src/datas/rbuffer.h /^typedef struct rbuffer_s_ {$/;" s +rbuffer_test ./src/datas/rbuffer.c /^void rbuffer_test(void)$/;" f +read_buf ./src/meta/meta.h /^ char* read_buf; \/**< Pointer to a read buffer *\/$/;" m struct:meta_s_ +read_pos ./src/datas/rbuffer.h /^ sig_atomic_t read_pos;$/;" m struct:rbuffer_s_ +renamed ./src/generate/vsize.h /^ bool renamed; \/**< True if file\/dir has been renamed *\/$/;" m struct:vsize_s_ +replay ./src/options.h /^ bool replay; \/**< If set ioreplay will run\/replay the test *\/$/;" m struct:options_s_ +replay_extract_header ./src/replay/replay.c /^void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes,$/;" f +replay_fd ./src/generate/generate.h /^ FILE *replay_fd; \/**< The fd of the .replay file *\/$/;" m struct:generate_s_ +replay_fd ./src/init/init.h /^ FILE *replay_fd;$/;" m struct:init_s_ +replay_fd ./src/meta/meta.h /^ FILE* replay_fd; \/**< The FS of the .replay file *\/$/;" m struct:meta_s_ +replay_file ./src/options.h /^ char *replay_file; \/**< The name of the .replay file *\/$/;" m struct:options_s_ +replay_run ./src/replay/replay.c /^status_e replay_run(options_s *opts)$/;" f +required ./src/generate/vsize.h /^ bool required; \/**< True if init mode will create this file\/dir *\/$/;" m struct:vsize_s_ +ret ./src/generate/gtask.h /^ int ret; \/**< ioreplay process status, SUCCESS if everything is alright *\/$/;" m struct:gtask_s_ +reuse_queue ./src/generate/generate.h /^ rbuffer_s *reuse_queue; \/**< A task buffer, for reusing these *\/$/;" m struct:generate_s_ +reuse_queue ./src/init/init.h /^ rbuffer_s *reuse_queue;$/;" m struct:init_s_ +reuse_queue_mutex ./src/init/init.h /^ pthread_mutex_t reuse_queue_mutex;$/;" m struct:init_s_ +right ./src/datas/btree.h /^ struct btreelem_ *right; \/**< The next element to the right *\/$/;" m struct:btreelem_ typeref:struct:btreelem_::btreelem_ +ring ./src/datas/rbuffer.h /^ void **ring;$/;" m struct:rbuffer_s_ +rioop_chmod ./src/replay/rioop.c /^void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_chown ./src/replay/rioop.c /^void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_close ./src/replay/rioop.c /^void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fchmod ./src/replay/rioop.c /^void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fchown ./src/replay/rioop.c /^void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fcntl ./src/replay/rioop.c /^void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fdatasync ./src/replay/rioop.c /^void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fstat ./src/replay/rioop.c /^void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fsync ./src/replay/rioop.c /^void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_getdents ./src/replay/rioop.c /^void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_lchown ./src/replay/rioop.c /^void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_lseek ./src/replay/rioop.c /^void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_mkdir ./src/replay/rioop.c /^void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_open ./src/replay/rioop.c /^void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_)$/;" f +rioop_read ./src/replay/rioop.c /^void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_rename ./src/replay/rioop.c /^void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_rmdir ./src/replay/rioop.c /^void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_run ./src/replay/rioop.c /^void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_stat ./src/replay/rioop.c /^void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_unlink ./src/replay/rioop.c /^void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_write ./src/replay/rioop.c /^void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +root ./src/datas/btree.h /^ btreelem_s *root; \/**< The root element *\/$/;" m struct:btree_s_ +rprocess_destroy ./src/replay/rprocess.c /^void rprocess_destroy(rprocess_s *p)$/;" f +rprocess_map ./src/replay/rworker.h /^ amap_s* rprocess_map; \/**< Holding all processes handled by this worker *\/$/;" m struct:__anon2 +rprocess_new ./src/replay/rprocess.c /^rprocess_s* rprocess_new(const int pid, amap_s *fds_map)$/;" f +rprocess_s ./src/replay/rprocess.h /^} rprocess_s;$/;" t typeref:struct:rprocess_s_ +rprocess_s_ ./src/replay/rprocess.h /^typedef struct rprocess_s_ {$/;" s +rtask_destroy ./src/replay/rtask.c /^void rtask_destroy(rtask_s *task)$/;" f +rtask_new ./src/replay/rtask.c /^rtask_s* rtask_new()$/;" f +rtask_s ./src/replay/rtask.h /^} rtask_s;$/;" t typeref:struct:rtask_s_ +rtask_s_ ./src/replay/rtask.h /^typedef struct rtask_s_ {$/;" s +rtask_update ./src/replay/rtask.c /^void rtask_update(rtask_s *task, void *worker, void *process, char *line,$/;" f +rthread_buffer ./src/replay/rworker.h /^ rbuffer_s *rthread_buffer; \/**< Buffering idle threads to be reused *\/$/;" m struct:__anon2 +rthread_buffer_mutex ./src/replay/rworker.h /^ pthread_mutex_t rthread_buffer_mutex; \/**< Sync access to rthread_buffer *\/$/;" m struct:__anon2 +rthread_destroy ./src/replay/rthread.c /^void rthread_destroy(rthread_s *t)$/;" f +rthread_fd ./src/replay/rthread.h /^ FILE *rthread_fd; \/**< Used for debugging purposes only *\/$/;" m struct:rthread_s_ +rthread_insert_task ./src/replay/rthread.c /^bool rthread_insert_task(rthread_s* t, rtask_s* task)$/;" f +rthread_map ./src/replay/rworker.h /^ amap_s* rthread_map; \/**< Holding all threads handled by this worker *\/$/;" m struct:__anon2 +rthread_new ./src/replay/rthread.c /^rthread_s* rthread_new(const long tid, void *worker)$/;" f +rthread_process_task ./src/replay/rthread.c /^void rthread_process_task(rthread_s* t, rtask_s *task,$/;" f +rthread_pthread_start ./src/replay/rthread.c /^void *rthread_pthread_start(void *data)$/;" f +rthread_s ./src/replay/rthread.h /^} rthread_s;$/;" t typeref:struct:rthread_s_ +rthread_s_ ./src/replay/rthread.h /^typedef struct rthread_s_ {$/;" s +rthread_terminate ./src/replay/rthread.c /^void rthread_terminate(rthread_s* t)$/;" f +rthread_update ./src/replay/rthread.c /^long rthread_update(rthread_s *t, const long tid)$/;" f +rworker_destroy ./src/replay/rworker.c /^void rworker_destroy(rworker_s *w)$/;" f +rworker_fd ./src/replay/rworker.h /^ FILE *rworker_fd; \/**< For debugging purposes only *\/$/;" m struct:__anon2 +rworker_new ./src/replay/rworker.c /^rworker_s* rworker_new(const int rworker_num, amap_s *fds_map,$/;" f +rworker_num ./src/replay/rprocess.h /^ int rworker_num; \/**< The worker number of the responsible worker *\/$/;" m struct:rprocess_s_ +rworker_num ./src/replay/rworker.h /^ int rworker_num; \/**< The current worker ID *\/$/;" m struct:__anon2 +rworker_process_lines ./src/replay/rworker.c /^status_e rworker_process_lines(rworker_s* w, const long num_lines)$/;" f +rworker_s ./src/replay/rworker.h /^} rworker_s;$/;" t typeref:struct:__anon2 +single_threaded ./src/replay/rthread.h /^ bool single_threaded; \/**< Worker is single threaded or not *\/$/;" m struct:rthread_s_ +size ./src/datas/amap.h /^ long size; \/**< The total size\/capacity of the amap *\/$/;" m struct:amap_s_ +size ./src/datas/btree.h /^ int size; \/**< The current size of the binary tree *\/$/;" m struct:btree_s_ +size ./src/datas/hmap.h /^ unsigned int size; \/**< Size of the hmap *\/$/;" m struct:hmap_s_ +size ./src/datas/rbuffer.h /^ int size;$/;" m struct:rbuffer_s_ +size ./src/datas/stack.h /^ unsigned long size; \/**< A count how many elements are in the stack *\/$/;" m struct:stack_s_ +sizes_created ./src/init/itask.h /^ long sizes_created;$/;" m struct:itask_s_ +speed_factor ./src/options.h /^ double speed_factor; \/**< Specifies how fast the test is replayed *\/$/;" m struct:options_s_ +stack_destroy ./src/datas/stack.c /^void stack_destroy(stack_s *s)$/;" f +stack_elem_s ./src/datas/stack.h /^} stack_elem_s;$/;" t typeref:struct:stack_elem_s_ +stack_elem_s_ ./src/datas/stack.h /^typedef struct stack_elem_s_ {$/;" s +stack_is_empty ./src/datas/stack.c /^int stack_is_empty(stack_s *s)$/;" f +stack_new ./src/datas/stack.c /^stack_s *stack_new()$/;" f +stack_new_reverse_from ./src/datas/stack.c /^stack_s* stack_new_reverse_from(stack_s *s)$/;" f +stack_pop ./src/datas/stack.c /^void* stack_pop(stack_s *s)$/;" f +stack_push ./src/datas/stack.c /^void stack_push(stack_s *s, void *data)$/;" f +stack_s ./src/datas/stack.h /^} stack_s;$/;" t typeref:struct:stack_s_ +stack_s_ ./src/datas/stack.h /^typedef struct stack_s_ {$/;" s +start_pthread ./src/utils/utils.c /^void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data)$/;" f +start_time ./src/generate/generate.h /^ long start_time; \/**< The start time from the .capture file *\/$/;" m struct:generate_s_ +status ./src/generate/gtask.h /^ int status; \/**< Operation\/syscall return status *\/$/;" m struct:gtask_s_ +status_e ./src/defaults.h /^} status_e;$/;" t typeref:enum:status_e_ +status_e_ ./src/defaults.h /^typedef enum status_e_ {$/;" g +strtok2_r ./src/utils/utils.c /^char* strtok2_r(char *str, char *delim, char **saveptr)$/;" f +strunquote ./src/utils/utils.c /^void strunquote(char *str)$/;" f +task_buffer ./src/replay/rworker.h /^ rbuffer_s *task_buffer; \/**< Buffering thread tasks to be reused *\/$/;" m struct:__anon2 +task_buffer_mutex ./src/replay/rworker.h /^ pthread_mutex_t task_buffer_mutex; \/**< To sync access to task_buffer *\/$/;" m struct:__anon2 +tasks ./src/replay/rthread.h /^ rbuffer_s* tasks; \/**< Holds all outstanding tasks *\/$/;" m struct:rthread_s_ +terminate ./src/generate/gparser.h /^ bool terminate; \/**< The parser thread will terminate if set to true *\/$/;" m struct:gparser_s_ +terminate ./src/generate/gwriter.h /^ bool terminate; \/**< The writer thread will terminate if set to true *\/$/;" m struct:gwriter_s_ +terminate ./src/init/ithread.h /^ bool terminate; \/**< Indicates that thread can terminate *\/$/;" m struct:ithread_s_ +terminate ./src/replay/rprocess.h /^ int terminate; \/**< Indicates whether the worker is terminating or not *\/$/;" m struct:rprocess_s_ +terminate ./src/replay/rthread.h /^ bool terminate; \/**< True if thread shall terminate *\/$/;" m struct:rthread_s_ +threads_map ./src/init/init.h /^ amap_s *threads_map;$/;" m struct:init_s_ +tid ./src/generate/gtask.h /^ long tid; \/**< The thread ID *\/$/;" m struct:gtask_s_ +tid ./src/replay/rthread.h /^ long tid; \/**< The virtual thread id *\/$/;" m struct:rthread_s_ +toks ./src/replay/rtask.h /^ char *toks[MAX_TOKENS+1]; \/**< The tokens parsed from the .replay line *\/$/;" m struct:rtask_s_ +top ./src/datas/stack.h /^ stack_elem_s *top; \/**< The top element of the stack, NULL if empty *\/$/;" m struct:stack_s_ +trash ./src/options.h /^ bool trash; \/**< If set ioreplay will trash the environment *\/$/;" m struct:options_s_ +unsure ./src/generate/vsize.h /^ bool unsure; \/**< True if the file type is not fully clear *\/$/;" m struct:vsize_s_ +updates ./src/generate/vsize.h /^ long updates; \/**< Amount of times this vsize has been updated *\/$/;" m struct:vsize_s_ +user ./src/options.h /^ char *user; \/**< The user name to run the test as *\/$/;" m struct:options_s_ +utests_run ./src/utests.c /^void utests_run()$/;" f +vfd ./src/generate/gtask.h /^ vfd_s *vfd; \/**< A pointer to the virtual file descriptor *\/$/;" m struct:gtask_s_ +vfd_buffer ./src/generate/generate.h /^ rbuffer_s *vfd_buffer; \/**< A virtual fd buffer, for reusing these *\/$/;" m struct:generate_s_ +vfd_destroy ./src/vfd.c /^void vfd_destroy(vfd_s *vfd)$/;" f +vfd_map ./src/generate/gprocess.h /^ hmap_s *vfd_map; \/**< All virtual file descriptors of that process *\/$/;" m struct:gprocess_s_ +vfd_new ./src/vfd.c /^vfd_s* vfd_new(const int fd, const long mapped_fd, char *path)$/;" f +vfd_print ./src/vfd.c /^void vfd_print(vfd_s *vfd)$/;" f +vfd_s ./src/vfd.h /^} vfd_s;$/;" t typeref:struct:vfd_s_ +vfd_s_ ./src/vfd.h /^typedef struct vfd_s_ {$/;" s +vfd_update ./src/vfd.c /^void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path)$/;" f +vsize ./src/generate/gtask.h /^ vsize_s *vsize; \/**< Pointer to the virtual size object *\/$/;" m struct:gtask_s_ +vsize ./src/generate/vsize.h /^ long vsize; \/**< The virtual size *\/$/;" m struct:vsize_s_ +vsize ./src/init/itask.h /^ long vsize;$/;" m struct:itask_s_ +vsize ./src/replay/rtask.h /^ unsigned long vsize; \/**< The vsize *\/$/;" m struct:rtask_s_ +vsize2 ./src/generate/gtask.h /^ vsize_s *vsize2; \/**< Pointer to a second virtual size object *\/$/;" m struct:gtask_s_ +vsize_adjust ./src/generate/vsize.c /^void vsize_adjust(vsize_s *v, vfd_s* vfd)$/;" f +vsize_close ./src/generate/vsize.c /^void vsize_close(vsize_s *v, void* vfd)$/;" f +vsize_deficit ./src/generate/vsize.h /^ long vsize_deficit; \/**< Size to use for file creating during init mode *\/$/;" m struct:vsize_s_ +vsize_destroy ./src/generate/vsize.c /^void vsize_destroy(vsize_s *v)$/;" f +vsize_map ./src/generate/generate.h /^ hmap_s *vsize_map; \/**< A hash map of all virtual size objects *\/$/;" m struct:generate_s_ +vsize_mkdir ./src/generate/vsize.c /^void vsize_mkdir(vsize_s *v, const char *path)$/;" f +vsize_new ./src/generate/vsize.c /^vsize_s* vsize_new(char *file_path, const unsigned long id,$/;" f +vsize_open ./src/generate/vsize.c /^void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags)$/;" f +vsize_read ./src/generate/vsize.c /^void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes)$/;" f +vsize_rename ./src/generate/vsize.c /^void vsize_rename(vsize_s *v, vsize_s *v2,$/;" f +vsize_rmdir ./src/generate/vsize.c /^void vsize_rmdir(vsize_s *v, const char *path)$/;" f +vsize_s ./src/generate/vsize.h /^} vsize_s;$/;" t typeref:struct:vsize_s_ +vsize_s_ ./src/generate/vsize.h /^typedef struct vsize_s_ {$/;" s +vsize_seek ./src/generate/vsize.c /^void vsize_seek(vsize_s *v, void *vfd, const long new_offset)$/;" f +vsize_stat ./src/generate/vsize.c /^void vsize_stat(vsize_s *v, const char *path)$/;" f +vsize_unlink ./src/generate/vsize.c /^void vsize_unlink(vsize_s *v, const char *path)$/;" f +vsize_write ./src/generate/vsize.c /^void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes)$/;" f +wd_base ./src/options.h /^ char *wd_base; \/**< The working directory base *\/$/;" m struct:options_s_ +whence ./src/generate/gtask.h /^ long whence; \/**< Whence *\/$/;" m struct:gtask_s_ +worker ./src/replay/rtask.h /^ void *worker; \/* The responsible worker object *\/$/;" m struct:rtask_s_ +worker ./src/replay/rthread.h /^ void *worker; \/**< The responsible worker object *\/$/;" m struct:rthread_s_ +write_pos ./src/datas/rbuffer.h /^ sig_atomic_t write_pos;$/;" m struct:rbuffer_s_ +writer ./src/generate/generate.h /^ struct gwriter_s_ *writer; \/**< A pointer to the writer object *\/$/;" m struct:generate_s_ typeref:struct:generate_s_::gwriter_s_ diff --git a/systemtap/Makefile b/systemtap/Makefile new file mode 100644 index 0000000..f027719 --- /dev/null +++ b/systemtap/Makefile @@ -0,0 +1,43 @@ +KERNEL ?= $(shell uname -r) +PROCESSOR ?= $(shell uname -p) +DESTDIR ?= /opt/ioreplay/systemtap/$(KERNEL) +UPDATEURI = http://debuginfo.centos.org/7/$(PROCESSOR) +DOWNLOADIR ?= ./downloads +all: prepare compile +prepare: + sed 's/execname() != "stapio"/pid() == target()/' ./src/ioreplay.stp > ./src/targetedioreplay.stp + sed 's/execname() != "stapio"/execname() == "java"/' ./src/ioreplay.stp > ./src/javaioreplay.stp +compile: + @echo Crosscompiling for Kernel version $(KERNEL) + for stp in ioreplay javaioreplay targetedioreplay; do \ + stap -v ./src/$$stp.stp -p 4 -r $(KERNEL) -m $$stp \ + -D MAXSTRINGLEN=255 -D MAXACTION=10000 -D MAXSKIPPED=10000\ + -g --suppress-time-limits --suppress-handler-errors; \ + done +testsystemtap: + stap -v -e 'probe vfs.read {printf("read performed\n"); exit()}' +clean: + @echo Cleaning modules + test -f ioreplay.ko && rm -v ioreplay.ko || exit 0 + test -f javaioreplay.ko && rm -v javaioreplay.ko || exit 0 + test -f targetedioreplay.ko && rm -v targetedioreplay.ko || exit 0 +install: + test -d $(DESTDIR) || mkdir -p $(DESTDIR) + test -f ioreplay.ko && cp -v ioreplay.ko $(DESTDIR)/ || exit 0 + test -f javaioreplay.ko && cp -v javaioreplay.ko $(DESTDIR)/ || exit 0 + test -f targetedioreplay.ko && cp -v targetedioreplay.ko $(DESTDIR)/ || exit 0 +uninstall: + test ! -z "$(DESTDIR)" && test -d $(DESTDIR)/ && find $(DESTDIR) -name \*.ko -delete || exit 0 +deinstall: uninstall +debuginfodownload: + test -d $(DOWNLOADIR) || mkdir -p $(DOWNLOADIR) + test -f $(DOWNLOADIR)/kernel-debuginfo-$(KERNEL).rpm || \ + wget -P $(DOWNLOADIR) $(UPDATEURI)/kernel-debuginfo-$(KERNEL).rpm + test -f $(DOWNLOADIR)/kernel-debuginfo-common-$(PROCESSOR)-$(KERNEL).rpm || \ + wget -P $(DOWNLOADIR) $(UPDATEURI)/kernel-debuginfo-common-$(PROCESSOR)-$(KERNEL).rpm +debuginfolocalinstall: + #yum remove kernel-debuginfo kernel-debuginfo-common-$(PROCESSOR) + yum localinstall $(DOWNLOADIR)/kernel-debuginfo-common-$(PROCESSOR)-$(KERNEL).rpm \ + $(DOWNLOADIR)/kernel-debuginfo-$(KERNEL).rpm +todo: + fgrep TODO ./src/* diff --git a/systemtap/src/ioreplay.stp b/systemtap/src/ioreplay.stp new file mode 100644 index 0000000..c4b4f0b --- /dev/null +++ b/systemtap/src/ioreplay.stp @@ -0,0 +1,591 @@ +#!/usr/bin/env stap + +# Copyright 2018 Mimecast Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used to capture I/O on a Linux based system in order to replay +# the same I/O via the ioreplay command line utility. +# +# The tool will generate one line per captured I/O syscall to a .capture log +# file. +# +# The key/value separator is ';:,', example line: +# +# t=1509989598023;:,i=17159:17358;:,o=open;:,d=167;:,p=/tmp/test;:,f=0;:,m=438;:, +# +# It may be that SystemTap will skip probes or interrupts probes in case of +# system overload. As a result we can have corrupt lines in the log. That's why +# we use a special field separator ';:,' to detect corrupt lines more robustly. +# +# The line uses the following format keys (we use many different of these, the +# only benefit over a more generic approach is to detect corrupt lines more +# easily): +# +# Format keys: +# t: Time +# i: PID:TID (process and thread ID) +# o: Operation name +# O: Offset or owner/user UID +# W: Whence +# d: File/dir descriptor +# p: File path +# P: File path 2 +# f: Flags +# m: Mode +# b: Bytes +# c: Count +# s: Return status +# t: Optional text +# F: FCNTL command +# G: FCNTL arg or user group UID +# T: Optional text (debugging purpose only) +# a: Address +# A: Address 2 +# + +# Return the full qualified version of path +function absolute_path (path) { + # Is it already a full qualified path? + if (substr(path,0,1) == "/") { + return path; + } + + # Look into the in Kernel task structure to look up the corresponding + # mount point and directory entry... + tc = task_current() + pwd_dentry = @cast(tc, "task_struct")->fs->pwd->dentry + pwd_mnt = @cast(tc, "task_struct")->fs->pwd->mnt + + # Construct a full qualified path from it! + return task_dentry_path(tc, pwd_dentry, pwd_mnt) . "/" . path; +} + +probe syscall.open.return, syscall.openat.return { + if (execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($flags), + @entry($mode)); + } +} + +probe syscall.lseek.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%d;:,W=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($whence), + $return); + } +} + +probe syscall.fcntl.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,F=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($cmd), + @entry($arg), + $return); + } +} + +probe syscall.creat.return { + if (execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($mode)); + } +} + +probe syscall.write.return, syscall.writev.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.unlink.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.unlinkat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.rename.return, syscall.renameat.return, syscall.renameat2.return { + if(execname() != "stapio") { + oldname = user_string(@entry($oldname)) + newname = user_string(@entry($newname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,P=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(oldname), + absolute_path(newname), + $return); + } +} + +probe syscall.read.return, syscall.readv.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readahead.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%ld;:,O=%ld;:,c=%ld\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return, + @entry($offset), + @entry($count)); + } +} + +probe syscall.readdir.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readlink.return { + if(execname() != "stapio") { + pathname = user_string(@entry($path)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.readlinkat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fdatasync.return, syscall.fsync.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.sync_file_range.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($nbytes), + $return); + } +} + +probe syscall.sync.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return); + } +} + +probe syscall.syncfs.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.close.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.getdents.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,c=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($count), + $return); + } +} + +probe syscall.mkdir.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.rmdir.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.mkdirat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.stat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.statfs.return, syscall.statfs64.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstatfs.return, syscall.fstatfs64.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.lstat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstat.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.fstatat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.chmod.return, syscall.fchmodat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.fchmod.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($mode), + $return); + } +} + +probe syscall.chown.return, syscall.chown16.return, + syscall.lchown.return, syscall.lchown16.return { + if(execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchown.return, syscall.fchown16.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchownat.return { + pathname = user_string(@entry($filename)) + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + @entry($flag), + $return); + } +} + +probe syscall.mmap2.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,m=%d;:,f=%d;:,d=%d;:,O=%ld;:,A=%ld;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + @entry($prot), + @entry($flags), + @entry($fd), + @entry($pgoff), + $return); + } +} + +probe syscall.mremap.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,A=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($new_addr), + @entry($addr), + @entry($new_len), + @entry($flags), + $return); + } +} + +probe syscall.munmap.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + $return); + } +} + +probe syscall.msync.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($start), + @entry($len), + @entry($flags), + $return); + } +} + +probe syscall.exit_group { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name); + } +} + +# Stop probing after 1h (for safety) +probe timer.s(3600) { + exit(); +} + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/systemtap/src/javaioreplay.stp b/systemtap/src/javaioreplay.stp new file mode 100644 index 0000000..9161f9e --- /dev/null +++ b/systemtap/src/javaioreplay.stp @@ -0,0 +1,596 @@ +#!/usr/bin/env stap + +# Copyright 2018 Mimecast Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used to capture I/O on a Linux based system in order to replay +# the same I/O via the ioreplay command line utility. +# +# The tool will generate one line per captured I/O syscall to a .capture log +# file. +# +# The key/value separator is ';:,', example line: +# +# t=1509989598023;:,i=17159:17358;:,o=open;:,d=167;:,p=/tmp/test;:,f=0;:,m=438;:, +# +# It may be that SystemTap will skip probes or interrupts probes in case of +# system overload. As a result we can have corrupt lines in the log. That's why +# we use a special field separator ';:,' to detect corrupt lines more robustly. +# +# The line uses the following format keys (we use many different of these, the +# only benefit over a more generic approach is to detect corrupt lines more +# easily): +# +# Format keys: +# t: Time +# i: PID:TID (process and thread ID) +# o: Operation name +# O: Offset or owner/user UID +# W: Whence +# d: File/dir descriptor +# p: File path +# P: File path 2 +# f: Flags +# m: Mode +# b: Bytes +# c: Count +# s: Return status +# t: Optional text +# F: FCNTL command +# G: FCNTL arg or user group UID +# T: Optional text (debugging purpose only) +# a: Address +# A: Address 2 +# + +# Return the full qualified version of path +function absolute_path (path) { + # Is it already a full qualified path? + if (substr(path,0,1) == "/") { + return path; + } + + # Look into the in Kernel task structure to look up the corresponding + # mount point and directory entry... + tc = task_current() + pwd_dentry = @cast(tc, "task_struct")->fs->pwd->dentry + pwd_mnt = @cast(tc, "task_struct")->fs->pwd->mnt + + # Construct a full qualified path from it! + return task_dentry_path(tc, pwd_dentry, pwd_mnt) . "/" . path; +} + +probe syscall.open.return, syscall.openat.return { + if (execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($flags), + @entry($mode)); + } +} + +probe syscall.lseek.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%d;:,W=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($whence), + $return); + } +} + +probe syscall.fcntl.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,F=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($cmd), + @entry($arg), + $return); + } +} + +probe syscall.creat.return { + if (execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($mode)); + } +} + +probe syscall.write.return, syscall.writev.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.unlink.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.unlinkat.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.rename.return, syscall.renameat.return, syscall.renameat2.return { + if(execname() == "java") { + oldname = user_string(@entry($oldname)) + newname = user_string(@entry($newname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,P=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(oldname), + absolute_path(newname), + $return); + } +} + +probe syscall.read.return, syscall.readv.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readahead.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%ld;:,O=%ld;:,c=%ld\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return, + @entry($offset), + @entry($count)); + } +} + +probe syscall.readdir.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readlink.return { + if(execname() == "java") { + pathname = user_string(@entry($path)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.readlinkat.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fdatasync.return, syscall.fsync.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.sync_file_range.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($nbytes), + $return); + } +} + +probe syscall.sync.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return); + } +} + +probe syscall.syncfs.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.close.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.getdents.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,c=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($count), + $return); + } +} + +probe syscall.mkdir.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.rmdir.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.mkdirat.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.stat.return { + if(execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.statfs.return, syscall.statfs64.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstatfs.return, syscall.fstatfs64.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.lstat.return { + if(execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstat.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.fstatat.return { + if(execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.chmod.return, syscall.fchmodat.return { + if(execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.fchmod.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($mode), + $return); + } +} + +probe syscall.chown.return, syscall.chown16.return, + syscall.lchown.return, syscall.lchown16.return { + if(execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchown.return, syscall.fchown16.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchownat.return { + pathname = user_string(@entry($filename)) + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + @entry($flag), + $return); + } +} + +probe syscall.mmap2.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,m=%d;:,f=%d;:,d=%d;:,O=%ld;:,A=%ld;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + @entry($prot), + @entry($flags), + @entry($fd), + @entry($pgoff), + $return); + } +} + +probe syscall.mremap.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,A=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($new_addr), + @entry($addr), + @entry($new_len), + @entry($flags), + $return); + } +} + +probe syscall.munmap.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + $return); + } +} + +probe syscall.msync.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($start), + @entry($len), + @entry($flags), + $return); + } +} + +probe syscall.exit_group { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name); + } +} + +# Stop probing after 1h (for safety) +probe timer.s(3600) { + exit(); +} + +# Stop probing after 1h (for safety) +probe timer.s(13) { + exit(); +} + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/systemtap/src/targetedioreplay.stp b/systemtap/src/targetedioreplay.stp new file mode 100644 index 0000000..aedee28 --- /dev/null +++ b/systemtap/src/targetedioreplay.stp @@ -0,0 +1,596 @@ +#!/usr/bin/env stap + +# Copyright 2018 Mimecast Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used to capture I/O on a Linux based system in order to replay +# the same I/O via the ioreplay command line utility. +# +# The tool will generate one line per captured I/O syscall to a .capture log +# file. +# +# The key/value separator is ';:,', example line: +# +# t=1509989598023;:,i=17159:17358;:,o=open;:,d=167;:,p=/tmp/test;:,f=0;:,m=438;:, +# +# It may be that SystemTap will skip probes or interrupts probes in case of +# system overload. As a result we can have corrupt lines in the log. That's why +# we use a special field separator ';:,' to detect corrupt lines more robustly. +# +# The line uses the following format keys (we use many different of these, the +# only benefit over a more generic approach is to detect corrupt lines more +# easily): +# +# Format keys: +# t: Time +# i: PID:TID (process and thread ID) +# o: Operation name +# O: Offset or owner/user UID +# W: Whence +# d: File/dir descriptor +# p: File path +# P: File path 2 +# f: Flags +# m: Mode +# b: Bytes +# c: Count +# s: Return status +# t: Optional text +# F: FCNTL command +# G: FCNTL arg or user group UID +# T: Optional text (debugging purpose only) +# a: Address +# A: Address 2 +# + +# Return the full qualified version of path +function absolute_path (path) { + # Is it already a full qualified path? + if (substr(path,0,1) == "/") { + return path; + } + + # Look into the in Kernel task structure to look up the corresponding + # mount point and directory entry... + tc = task_current() + pwd_dentry = @cast(tc, "task_struct")->fs->pwd->dentry + pwd_mnt = @cast(tc, "task_struct")->fs->pwd->mnt + + # Construct a full qualified path from it! + return task_dentry_path(tc, pwd_dentry, pwd_mnt) . "/" . path; +} + +probe syscall.open.return, syscall.openat.return { + if (pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($flags), + @entry($mode)); + } +} + +probe syscall.lseek.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%d;:,W=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($whence), + $return); + } +} + +probe syscall.fcntl.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,F=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($cmd), + @entry($arg), + $return); + } +} + +probe syscall.creat.return { + if (pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($mode)); + } +} + +probe syscall.write.return, syscall.writev.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.unlink.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.unlinkat.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.rename.return, syscall.renameat.return, syscall.renameat2.return { + if(pid() == target()) { + oldname = user_string(@entry($oldname)) + newname = user_string(@entry($newname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,P=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(oldname), + absolute_path(newname), + $return); + } +} + +probe syscall.read.return, syscall.readv.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readahead.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%ld;:,O=%ld;:,c=%ld\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return, + @entry($offset), + @entry($count)); + } +} + +probe syscall.readdir.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readlink.return { + if(pid() == target()) { + pathname = user_string(@entry($path)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.readlinkat.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fdatasync.return, syscall.fsync.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.sync_file_range.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($nbytes), + $return); + } +} + +probe syscall.sync.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return); + } +} + +probe syscall.syncfs.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.close.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.getdents.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,c=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($count), + $return); + } +} + +probe syscall.mkdir.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.rmdir.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.mkdirat.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.stat.return { + if(pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.statfs.return, syscall.statfs64.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstatfs.return, syscall.fstatfs64.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.lstat.return { + if(pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstat.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.fstatat.return { + if(pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.chmod.return, syscall.fchmodat.return { + if(pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.fchmod.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($mode), + $return); + } +} + +probe syscall.chown.return, syscall.chown16.return, + syscall.lchown.return, syscall.lchown16.return { + if(pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchown.return, syscall.fchown16.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchownat.return { + pathname = user_string(@entry($filename)) + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + @entry($flag), + $return); + } +} + +probe syscall.mmap2.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,m=%d;:,f=%d;:,d=%d;:,O=%ld;:,A=%ld;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + @entry($prot), + @entry($flags), + @entry($fd), + @entry($pgoff), + $return); + } +} + +probe syscall.mremap.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,A=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($new_addr), + @entry($addr), + @entry($new_len), + @entry($flags), + $return); + } +} + +probe syscall.munmap.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + $return); + } +} + +probe syscall.msync.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($start), + @entry($len), + @entry($flags), + $return); + } +} + +probe syscall.exit_group { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name); + } +} + +# Stop probing after 1h (for safety) +probe timer.s(3600) { + exit(); +} + +# Stop probing after 1h (for safety) +probe timer.s(13) { + exit(); +} + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 |
