imacat/arclog
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added support for the xz compression.

Browse Source
This commit is contained in:
依瑪貓
2022-03-19 00:45:17 +08:00
parent 3c7030099b
commit 5b8d678b6b
8 changed files with 727 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
t/01-exhaust.t
View File

@ -1,7 +1,7 @@
#! /usr/bin/perl -w
# Test all the possible combination of options
# Copyright (c) 2007-2021 imacat.
# Copyright (c) 2007-2022 imacat.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -21,7 +21,7 @@ use warnings;
use diagnostics;
use Test;
BEGIN { plan tests => 2160 }
BEGIN { plan tests => 3600 }
use File::Basename qw(basename);
use File::Path qw(mkpath rmtree);
@ -35,7 +35,7 @@ $WORKDIR = catdir($FindBin::Bin, "logs");
$arclog = catfile($FindBin::Bin, updir, "blib", "script", "arclog");
$tno = 0;
# 1-2160: All possible option combinations
# 1-3600: All possible option combinations
# Test each log file format
foreach my $fmt (@LOG_FORMATS) {
# Test each source log file type

33
t/02-several.t
View File

@ -1,7 +1,7 @@
#! /usr/bin/perl -w
# Test archiving several log files at once
# Copyright (c) 2007-2021 imacat.
# Copyright (c) 2007-2022 imacat.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -21,7 +21,7 @@ use warnings;
use diagnostics;
use Test;
BEGIN { plan tests => 16 }
BEGIN { plan tests => 25 }
use File::Basename qw(basename);
use File::Path qw(mkpath rmtree);
@ -35,14 +35,15 @@ $WORKDIR = catdir($FindBin::Bin, "logs");
$arclog = catfile($FindBin::Bin, updir, "blib", "script", "arclog");
$tno = 0;
# 1-16: Archiving several log files at once
# 1-25: Archiving several log files at once
foreach my $rt (@RESULT_TYPES) {
my $skip;
$skip = 0;
# 1: Source log files listed as the arguments
$_ = eval {
if ( ($$rt{"type"} eq TYPE_GZIP && has_no_gzip)
|| ($$rt{"type"} eq TYPE_BZIP2 && has_no_bzip2)) {
|| ($$rt{"type"} eq TYPE_BZIP2 && has_no_bzip2)
|| ($$rt{"type"} eq TYPE_XZ && has_no_xz)) {
$skip = 1;
return;
}
@ -56,13 +57,15 @@ foreach my $rt (@RESULT_TYPES) {
$title = join ", ", "several log files", "all listed as arguments",
$$fmt{"title"}, $$rt{"title"};
# (2-4 times available compression) log files
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2);
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2)
+ (has_no_xz? 0: 2);
$num = $_ + int rand $_;
my %types = qw();
# At least 2 files for each available compression
foreach my $st (@SOURCE_TYPES) {
next if ($$st{"type"} eq TYPE_GZIP && has_no_gzip)
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2);
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2)
|| ($$st{"type"} eq TYPE_XZ && has_no_xz);
@_ = grep !exists $types{$_}, (0...$num-1);
$types{$_[int rand @_]} = $st;
@_ = grep !exists $types{$_}, (0...$num-1);
@ -73,7 +76,8 @@ foreach my $rt (@RESULT_TYPES) {
do {
$types{$_} = $SOURCE_TYPES[int rand @SOURCE_TYPES];
} until !(${$types{$_}}{"type"} eq TYPE_GZIP && has_no_gzip)
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2);
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2)
&& !(${$types{$_}}{"type"} eq TYPE_XZ && has_no_xz);
}
@st = map $types{$_}, (0...$num-1);
@fs = qw();
@ -146,15 +150,17 @@ foreach my $rt (@RESULT_TYPES) {
skip($skip, $_, 1, $@);
clean_up $_ || $skip, $WORKDIR, ++$tno;
# 2-4: One of the source log files is read from STDIN
# 2-5: One of the source log files is read from STDIN
# The file type at STDIN
foreach my $st_stdin (@SOURCE_TYPES) {
$skip = 0;
$_ = eval {
if ( ($$rt{"type"} eq TYPE_GZIP && has_no_gzip)
|| ($$rt{"type"} eq TYPE_BZIP2 && has_no_bzip2)
|| ($$rt{"type"} eq TYPE_XZ && has_no_xz)
|| ($$st_stdin{"type"} eq TYPE_GZIP && has_no_gzip)
|| ($$st_stdin{"type"} eq TYPE_BZIP2 && has_no_bzip2)) {
|| ($$st_stdin{"type"} eq TYPE_BZIP2 && has_no_bzip2)
|| ($$st_stdin{"type"} eq TYPE_XZ && has_no_xz)) {
$skip = 1;
return;
}
@ -168,13 +174,15 @@ foreach my $rt (@RESULT_TYPES) {
$title = join ", ", "several log files", "one read from STDIN",
"STDIN " . $$st_stdin{"title"}, $$rt{"title"};
# (2-4 times available compression) log files
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2);
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2)
+ (has_no_xz? 0: 2);
$num = $_ + int rand $_;
my %types = qw();
# At least 2 files for each available compression
foreach my $st (@SOURCE_TYPES) {
next if ($$st{"type"} eq TYPE_GZIP && has_no_gzip)
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2);
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2)
|| ($$st{"type"} eq TYPE_XZ && has_no_xz);
@_ = grep !exists $types{$_}, (0...$num-1);
$types{$_[int rand @_]} = $st;
@_ = grep !exists $types{$_}, (0...$num-1);
@ -185,7 +193,8 @@ foreach my $rt (@RESULT_TYPES) {
do {
$types{$_} = $SOURCE_TYPES[int rand @SOURCE_TYPES];
} until !(${$types{$_}}{"type"} eq TYPE_GZIP && has_no_gzip)
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2);
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2)
&& !(${$types{$_}}{"type"} eq TYPE_XZ && has_no_xz);
}
# Choose the STDIN from the matching compression
@_ = grep ${$types{$_}}{"type"} eq $$st_stdin{"type"}, (0...$num-1);

26
t/03-fallback.t
View File

@ -1,7 +1,7 @@
#! /usr/bin/perl -w
# Test fallback behavior
# Copyright (c) 2007-2021 imacat.
# Copyright (c) 2007-2022 imacat.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -50,18 +50,21 @@ foreach my $kt (@KEEP_MODES) {
do {
$rt = $RESULT_TYPES[int rand @RESULT_TYPES];
} until !($$rt{"type"} eq TYPE_GZIP && has_no_gzip)
&& !($$rt{"type"} eq TYPE_BZIP2 && has_no_bzip2);
&& !($$rt{"type"} eq TYPE_BZIP2 && has_no_bzip2)
&& !($$rt{"type"} eq TYPE_XZ && has_no_xz);
$title = join ", ", "STDIN keep fall back", $$kt{"title"},
$$fmt{"title"}, $$rt{"title"};
# (2-4 times available compression) log files
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2);
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2)
+ (has_no_xz? 0: 2);
$num = $_ + int rand $_;
$stdin = int rand $num;
my %types = qw();
# At least 2 files for each available compression
foreach my $st (@SOURCE_TYPES) {
next if ($$st{"type"} eq TYPE_GZIP && has_no_gzip)
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2);
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2)
|| ($$st{"type"} eq TYPE_XZ && has_no_xz);
@_ = grep !exists $types{$_}, (0...$num-1);
$types{$_[int rand @_]} = $st;
@_ = grep !exists $types{$_}, (0...$num-1);
@ -72,7 +75,8 @@ foreach my $kt (@KEEP_MODES) {
do {
$types{$_} = $SOURCE_TYPES[int rand @SOURCE_TYPES];
} until !(${$types{$_}}{"type"} eq TYPE_GZIP && has_no_gzip)
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2);
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2)
&& !(${$types{$_}}{"type"} eq TYPE_XZ && has_no_xz);
}
@st = map $types{$_}, (0...$num-1);
@fs = qw();
@ -160,18 +164,21 @@ $_ = eval {
do {
$rt = $RESULT_TYPES[int rand @RESULT_TYPES];
} until !($$rt{"type"} eq TYPE_GZIP && has_no_gzip)
&& !($$rt{"type"} eq TYPE_BZIP2 && has_no_bzip2);
&& !($$rt{"type"} eq TYPE_BZIP2 && has_no_bzip2)
&& !($$rt{"type"} eq TYPE_XZ && has_no_xz);
$title = join ", ", "STDIN override ask fall back",
$$fmt{"title"}, $$rt{"title"};
# (2-4 times available compression) log files
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2);
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2)
+ (has_no_xz? 0: 2);
$num = $_ + int rand $_;
$stdin = int rand $num;
my %types = qw();
# At least 2 files for each available compression
foreach my $st (@SOURCE_TYPES) {
next if ($$st{"type"} eq TYPE_GZIP && has_no_gzip)
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2);
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2)
|| ($$st{"type"} eq TYPE_XZ && has_no_xz);
@_ = grep !exists $types{$_}, (0...$num-1);
$types{$_[int rand @_]} = $st;
@_ = grep !exists $types{$_}, (0...$num-1);
@ -182,7 +189,8 @@ $_ = eval {
do {
$types{$_} = $SOURCE_TYPES[int rand @SOURCE_TYPES];
} until !(${$types{$_}}{"type"} eq TYPE_GZIP && has_no_gzip)
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2);
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2)
&& !(${$types{$_}}{"type"} eq TYPE_XZ && has_no_xz);
}
@st = map $types{$_}, (0...$num-1);
@fs = qw();

11
t/04-errors.t
View File

@ -1,7 +1,7 @@
#! /usr/bin/perl -w
# Test the errors that should be captured.
# Copyright (c) 2007-2021 imacat.
# Copyright (c) 2007-2022 imacat.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -140,13 +140,15 @@ $_ = eval {
$title = join ",", "A same log file is specified more than once",
$$fmt{"title"}, $$rt{"title"};
# (2-4 times available compression) log files
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2);
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2)
+ (has_no_xz? 0: 2);
$num = $_ + int rand $_;
my %types = qw();
# At least 2 files for each available compression
foreach my $st (@SOURCE_TYPES) {
next if ($$st{"type"} eq TYPE_GZIP && has_no_gzip)
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2);
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2)
|| ($$st{"type"} eq TYPE_XZ && has_no_xz);
@_ = grep !exists $types{$_}, (0...$num-1);
$types{$_[int rand @_]} = $st;
@_ = grep !exists $types{$_}, (0...$num-1);
@ -157,7 +159,8 @@ $_ = eval {
do {
$types{$_} = $SOURCE_TYPES[int rand @SOURCE_TYPES];
} until !(${$types{$_}}{"type"} eq TYPE_GZIP && has_no_gzip)
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2);
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2)
&& !(${$types{$_}}{"type"} eq TYPE_XZ && has_no_xz);
}
@st = map $types{$_}, (0...$num-1);
@fs = qw();

87
t/_helper.pm
View File

@ -1,6 +1,6 @@
# _helper.pm - A simple test suite helper
# Copyright (c) 2007-2021 imacat.
# Copyright (c) 2007-2022 imacat.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -24,12 +24,12 @@ $VERSION = "0.01";
@EXPORT = qw(
read_file read_raw_file write_file write_raw_file
run_cmd where_is file_type list_files preserve_source clean_up
has_no_file has_no_gzip has_no_bzip2
has_no_file has_no_gzip has_no_bzip2 has_no_xz
make_log_file
make_apache_log_file make_syslog_log_file
make_ntp_log_file make_apache_ssl_log_file make_modified_iso_log_file
random_word
TYPE_TEXT TYPE_GZIP TYPE_BZIP2
TYPE_TEXT TYPE_GZIP TYPE_BZIP2 TYPE_XZ
@LOG_FORMATS @SOURCE_TYPES @RESULT_TYPES @KEEP_MODES @OVERRIDE_MODES);
# Prototype declaration
sub this_file();
@ -46,6 +46,7 @@ sub clean_up($$$);
sub has_no_file();
sub has_no_gzip();
sub has_no_bzip2();
sub has_no_xz();
sub make_log_file($$$@);
sub make_apache_log_file($;$);
sub make_syslog_log_file($;$);
@ -71,15 +72,17 @@ use File::Temp qw(tempfile);
use Time::Local qw(timelocal);
$Data::Dumper::Indent = 1;
our (%WHERE_IS, $HAS_NO_FILE, $HAS_NO_GZIP, $HAS_NO_BZIP2);
our (%WHERE_IS, $HAS_NO_FILE, $HAS_NO_GZIP, $HAS_NO_BZIP2, $HAS_NO_XZ);
%WHERE_IS = qw();
undef $HAS_NO_FILE;
undef $HAS_NO_GZIP;
undef $HAS_NO_BZIP2;
undef $HAS_NO_XZ;
use constant TYPE_TEXT => "text/plain";
use constant TYPE_GZIP => "application/x-gzip";
use constant TYPE_BZIP2 => "application/x-bzip2";
use constant TYPE_XZ => "application/x-xz";
our (@LOG_FORMATS, @SOURCE_TYPES, @RESULT_TYPES, @KEEP_MODES, @OVERRIDE_MODES);
# All the log format information
@ -107,7 +110,11 @@ our (@LOG_FORMATS, @SOURCE_TYPES, @RESULT_TYPES, @KEEP_MODES, @OVERRIDE_MODES);
{ "title" => "bzip2 source",
"type" => TYPE_BZIP2,
"suf" => ".bz2",
"skip" => has_no_bzip2, }, );
"skip" => has_no_bzip2, },
{ "title" => "xz source",
"type" => TYPE_XZ,
"suf" => ".xz",
"skip" => has_no_xz, }, );
# All the result type information
@RESULT_TYPES = (
{ "title" => "default compress",
@ -125,6 +132,11 @@ our (@LOG_FORMATS, @SOURCE_TYPES, @RESULT_TYPES, @KEEP_MODES, @OVERRIDE_MODES);
"suf" => ".bz2",
"skip" => has_no_bzip2,
"opts" => [qw(-c b)], },
{ "title" => "xz compress",
"type" => TYPE_XZ,
"suf" => ".xz",
"skip" => has_no_xz,
"opts" => [qw(-c x)], },
{ "title" => "no compress",
"type" => TYPE_TEXT,
"suf" => "",
@ -275,6 +287,36 @@ sub read_file($) {
return $content;
}
# an xz compressed file
} elsif ($file =~ /\.xz/) {
# IO::Uncompress::UnXz
if (eval { require IO::Uncompress::UnXz; 1; }) {
my ($FH, $xz);
$content = "";
open $FH, $file or die this_file . ": $file: $!";
$xz = IO::Uncompress::UnXz->new($FH)
or die this_file . ": $file: $IO::Uncompress::UnXz::UnXzError";
while (1) {
($xz->read($_, 10240) != -1)
or die this_file . ": $file: $IO::Uncompress::UnXz::UnXzError";
$content .= $_;
last if length $_ < 10240;
}
$xz->close or die this_file . ": $file: $IO::Uncompress::UnXz::UnXzError";
return $content;
# xz executable
} else {
my ($PH, $CMD);
$CMD = where_is "xz";
$CMD = "xz -cd \"$file\"";
open $PH, "$CMD |" or die this_file . ": $CMD: $!";
$content = join "", <$PH>;
close $PH or die this_file . ": $CMD: $!";
return $content;
}
# a plain text file
} else {
my $FH;
@ -360,6 +402,30 @@ sub write_file($$) {
return;
}
# an xz compressed file
} elsif ($file =~ /\.xz/) {
# IO::Compress::Xz
if (eval { require IO::Compress::Xz; 1; }) {
my ($FH, $xz);
open $FH, ">$file" or die this_file . ": $file: $!";
$xz = IO::Compress::Xz->new($FH)
or die this_file . ": $file: $IO::Compress::Xz::XzError";
($xz->write($content, length $content) == length $content)
or die this_file . ": $file: $IO::Compress::Xz::XzError";
$xz->close or die this_file . ": $file: $IO::Compress::Xz::XzError";
return;
# xz executable
} else {
my ($PH, $CMD);
$CMD = where_is "xz";
$CMD = "\"$CMD\" -9f > \"$file\"";
open $PH, "| $CMD" or die this_file . ": $CMD: $!";
print $PH $content or die this_file . ": $CMD: $!";
close $PH or die this_file . ": $CMD: $!";
return;
}
# a plain text file
} else {
my $FH;
@ -452,6 +518,7 @@ sub file_type($) {
$_ = File::MMagic->new->checktype_filename($file);
return "application/x-gzip" if /gzip/;
return "application/x-bzip2" if /bzip2/;
return "application/x-xz" if /xz/;
# All else are text/plain
return "text/plain";
}
@ -460,6 +527,7 @@ sub file_type($) {
$_ = join "", `"$_" "$file"`;
return "application/x-gzip" if /gzip/;
return "application/x-bzip2" if /bzip2/;
return "application/x-xz" if /: XZ/;
# All else are text/plain
return "text/plain";
}
@ -549,6 +617,15 @@ sub has_no_bzip2() {
return $HAS_NO_BZIP2;
}
# If we have xz support somewhere
sub has_no_xz() {
$HAS_NO_XZ = eval { require IO::Compress::Xz; require IO::Uncompress::UnXz; 1; }
|| defined where_is "xz"?
0: "IO::Compress::Xz or xz executable not available"
if !defined $HAS_NO_XZ;
return $HAS_NO_XZ;
}
# Create a random existing log file
sub make_log_file($$$@) {
local ($_, %_);