Testing with examples from https://en.cppreference.com/w/

[mingodad]

Using the Lua script shown bellow to extract the code examples from https://en.cppreference.com/w/ I'm getting this results:

#cxx
Total files scanned     :	5629
Total files with code   :	2889
Total files failed code :	172

#clang-17
Total files scanned     :	5629
Total files with code   :	2889
Total files failed code :	266

#g++-13
Total files scanned     :	5629
Total files with code   :	2889
Total files failed code :	197

#circle-200
Total files scanned     :	5629
Total files with code   :	2889
Total files failed code :	474

local base_dir = "/home/mingo/.local/share/Zeal/Zeal/docsets/C++.docset/Contents/Resources/Documents/en.cppreference.com/w/cpp/";

local cppCodeFName = "/tmp/cppCode.cpp";
--local cxx_cmd = "../build/src/frontend/cxx -fsyntax-only -toolchain linux " .. cppCodeFName;
--local cxx_cmd = "/usr/bin/time ../build-release/src/frontend/cxx -fsyntax-only -toolchain linux " .. cppCodeFName;
local cxx_cmd = "/usr/bin/time ./cxx -fsyntax-only -toolchain linux " .. cppCodeFName;
--local cxx_cmd = "clang-17-env /usr/bin/time clang++ -std=c++23 -fsyntax-only " .. cppCodeFName;
--local cxx_cmd = "clang-17-env /usr/bin/time clang++ -fsyntax-only " .. cppCodeFName;
--local cxx_cmd = "gcc-13-env /usr/bin/time g++ -std=c++23 -fsyntax-only " .. cppCodeFName;
--local cxx_cmd = "gcc-13-env /usr/bin/time g++ -fsyntax-only " .. cppCodeFName;
--local cxx_cmd = "/usr/bin/time timeout 3s circle  -std=c++2b -fsyntax-only " .. cppCodeFName;

local code_re = "<div class=\"t%-example%-live%-link\">.-<div class=\"cpp source%-cpp\">(.-)</pre></div>";

local find_cmd = "find " .. base_dir .. " -name '*.html'";
local start_t = os.time();

local html_list = {};
for fname in io.popen(find_cmd, "r"):lines("l") do
	table.insert(html_list, fname);
end
table.sort(html_list);

local count_with_code = 0;
local fname_with_code_failed = {};
for idx, fname in ipairs(html_list) do
	print(fname);

	local html = io.open(fname, "r"):read("a");
	for code in html:gmatch(code_re) do
		code = code:gsub("%b<>", "");
		code = code:gsub("&nbsp;", " ");
		code = code:gsub("&lt;", "<");
		code = code:gsub("&gt;", ">");
		code = code:gsub("&amp;", "&");

		--print("//====Start");
		--print(code);
		--print("//====End");

		local fd = io.open(cppCodeFName, "w");
		fd:write(code);
		fd:close();
		count_with_code = count_with_code + 1;
		print("==CHECK==")
		print(cxx_cmd);
		io.stdout:flush();
		local rc, rc_type, rc_code = os.execute(cxx_cmd);
		if rc_code == 0 then
			print("==rc:OK==")
		else
			table.insert(fname_with_code_failed, idx);
			print("==rc:FAILED==")
			print("//====Start");
			print(code);
			print("//====End");
		end
	end

end
local end_t = os.time();
print("elapsed time: ", os.difftime(end_t, start_t));
print("Total files scanned     :", #html_list);
print("Total files with code   :", count_with_code);
print("Total files failed code :", #fname_with_code_failed);
for idx, fname_idx in ipairs(fname_with_code_failed) do
	print(html_list[fname_idx]);
end

See attached the output for cxx and clang-17: test-cpp-reference-examples-cxx.output.zip test-cpp-reference-examples-clang-17-std23.output.zip

[mingodad]

Also testing https://github.com/abseil/abseil-cpp with the Lua script shown bellow gives this result:

#cxx
Total files scanned     :	172
Total files with code   :	0
Total files failed code :	47

#clang-17
Total files scanned     :	172
Total files with code   :	0
Total files failed code :	3

#g++-13
Total files scanned     :	172
Total files with code   :	0
Total files failed code :	71

#circle-200
Total files scanned     :	172
Total files with code   :	0
Total files failed code :	59

local base_dir = "/home/mingo/dev/c/A_libs/abseil-cpp/absl/";

--local cxx_cmd = "/usr/bin/time ../build/src/frontend/cxx -fsyntax-only -toolchain linux ";
local cxx_cmd = "/usr/bin/time ../build-release/src/frontend/cxx -fsyntax-only -toolchain linux ";
--local cxx_cmd = "clang-17-env /usr/bin/time clang++ -std=c++14 -fsyntax-only ";
--local cxx_cmd = "clang-17-env /usr/bin/time clang++ -fsyntax-only ";
--local cxx_cmd = "gcc-13-env /usr/bin/time g++ -std=c++14 -fsyntax-only ";
--local cxx_cmd = "gcc-13-env /usr/bin/time g++ -fsyntax-only ";
--local cxx_cmd = "/usr/bin/time timeout 3s circle -fsyntax-only ";


local find_cmd = "find " .. base_dir .. " -name '*.cc'";
local start_t = os.time();

local cpp_list = {};
for fname in io.popen(find_cmd, "r"):lines("l") do
	if not (fname:match("_test.cc$") or fname:match("_benchmark.cc$")) then
		table.insert(cpp_list, fname);
	end
end
table.sort(cpp_list);

local count_with_code = 0;
local fname_with_code_failed = {};
for idx, fname in ipairs(cpp_list) do
	print(fname);

	local cmd = cxx_cmd .. fname .. " -I" .. base_dir .. "..";
	print("==CHECK==")
	print(cmd);
	io.stdout:flush();
	local rc = os.execute(cmd);
	if rc then
		print("==rc:OK==")
	else
		table.insert(fname_with_code_failed, idx);
		print("==rc:FAILED==")
	end

end
local end_t = os.time();
print("elapsed time: ", os.difftime(end_t, start_t));
print("Total files scanned     :", #cpp_list);
print("Total files with code   :", count_with_code);
print("Total files failed code :", #fname_with_code_failed);
for idx, fname_idx in ipairs(fname_with_code_failed) do
	print(cpp_list[fname_idx]);
end

See attached cxx and clang output: test-abseil-cpp-cxx.output.zip test-abseil-cpp-clang17.output.zip

[robertoraggi]

This looks very helpful, thanks for sharing. I will have a look at the failures, do you have a repo for this?

[mingodad]

No repo only the Lua script and you can download the offline cpp-reference from here https://en.cppreference.com/w/Cppreference:Archives .