imacat/reslog
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added support for the xz compression.

Browse Source
This commit is contained in:
依瑪貓
2022-03-19 21:13:23 +08:00
parent fb37d091d6
commit 67dca3926a
9 changed files with 650 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
t/01-exhaust.t
View File

@ -1,7 +1,7 @@
#! /usr/bin/perl -w
# Test all the possible combination of options
# Copyright (c) 2005-2021 imacat.
# Copyright (c) 2005-2022 imacat.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -23,7 +23,7 @@ use warnings;
use diagnostics;
use Test;
BEGIN { plan tests => 1341 }
BEGIN { plan tests => 1788 }
use File::Basename qw(basename);
use File::Path qw(mkpath rmtree);
@ -193,7 +193,7 @@ foreach my $st (@SOURCE_TYPES) {
die unless $_ || $$st{"skip"};
}
# 85: From STDIN to STDOUT
# 149: From STDIN to STDOUT
$_ = eval {
return if $$st{"skip"};
my ($title, $cmd, $ret_no, $out, $err, $logfile, $result);

27
t/02-several.t
View File

@ -1,7 +1,7 @@
#! /usr/bin/perl -w
# Test processing several log files at once
# Copyright (c) 2007-2021 imacat.
# Copyright (c) 2007-2022 imacat.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -23,7 +23,7 @@ use warnings;
use diagnostics;
use Test;
BEGIN { plan tests => 4 }
BEGIN { plan tests => 5 }
use File::Basename qw(basename);
use File::Path qw(mkpath rmtree);
@ -48,13 +48,15 @@ $_ = eval {
$suf = "." . random_word;
do { $trim_suf = "." . random_word; } until $trim_suf ne $suf;
# (2-4 times available compression) log files
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2);
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2)
+ (has_no_xz? 0: 2);
$num = $_ + int rand $_;
my %types = qw();
# At least 2 files for each available compression
foreach my $st (@SOURCE_TYPES) {
next if ($$st{"type"} eq TYPE_GZIP && has_no_gzip)
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2);
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2)
|| ($$st{"type"} eq TYPE_XZ && has_no_xz);
@_ = grep !exists $types{$_}, (0...$num-1);
$types{$_[int rand @_]} = $st;
@_ = grep !exists $types{$_}, (0...$num-1);
@ -65,7 +67,8 @@ $_ = eval {
do {
$types{$_} = $SOURCE_TYPES[int rand @SOURCE_TYPES];
} until !(${$types{$_}}{"type"} eq TYPE_GZIP && has_no_gzip)
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2);
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2)
&& !(${$types{$_}}{"type"} eq TYPE_XZ && has_no_xz);
}
@st = map $types{$_}, (0...$num-1);
@fs = qw();
@ -122,14 +125,15 @@ $_ = eval {
ok($_, 1, $@);
clean_up $_, $WORKDIR, ++$tno;
# 2-4: One of the source log files is read from STDIN
# 2-5: One of the source log files is read from STDIN
# The file type at STDIN
foreach my $st_stdin (@SOURCE_TYPES) {
my $skip;
$skip = 0;
$_ = eval {
if ( ($$st_stdin{"type"} eq TYPE_GZIP && has_no_gzip)
|| ($$st_stdin{"type"} eq TYPE_BZIP2 && has_no_bzip2)) {
|| ($$st_stdin{"type"} eq TYPE_BZIP2 && has_no_bzip2)
|| ($$st_stdin{"type"} eq TYPE_XZ && has_no_xz)) {
$skip = 1;
return;
}
@ -143,13 +147,15 @@ foreach my $st_stdin (@SOURCE_TYPES) {
$suf = "." . random_word;
do { $trim_suf = "." . random_word; } until $trim_suf ne $suf;
# (2-4 times available compression) log files
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2);
$_ = 2 + (has_no_gzip? 0: 2) + (has_no_bzip2? 0: 2)
+ (has_no_xz? 0: 2);
$num = $_ + int rand $_;
my %types = qw();
# At least 2 files for each available compression
foreach my $st (@SOURCE_TYPES) {
next if ($$st{"type"} eq TYPE_GZIP && has_no_gzip)
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2);
|| ($$st{"type"} eq TYPE_BZIP2 && has_no_bzip2)
|| ($$st{"type"} eq TYPE_XZ && has_no_xz);
@_ = grep !exists $types{$_}, (0...$num-1);
$types{$_[int rand @_]} = $st;
@_ = grep !exists $types{$_}, (0...$num-1);
@ -160,7 +166,8 @@ foreach my $st_stdin (@SOURCE_TYPES) {
do {
$types{$_} = $SOURCE_TYPES[int rand @SOURCE_TYPES];
} until !(${$types{$_}}{"type"} eq TYPE_GZIP && has_no_gzip)
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2);
&& !(${$types{$_}}{"type"} eq TYPE_BZIP2 && has_no_bzip2)
&& !(${$types{$_}}{"type"} eq TYPE_XZ && has_no_xz);
}
# Choose the STDIN from the matching compression
@_ = grep ${$types{$_}}{"type"} eq $$st_stdin{"type"}, (0...$num-1);

8
t/03-errors.t
View File

@ -1,7 +1,7 @@
#! /usr/bin/perl -w
# Test the errors that should be captured.
# Copyright (c) 2007-2021 imacat.
# Copyright (c) 2007-2022 imacat.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -21,7 +21,7 @@ use warnings;
use diagnostics;
use Test;
BEGIN { plan tests => 7 }
BEGIN { plan tests => 9 }
use File::Basename qw(basename);
use File::Path qw(mkpath rmtree);
@ -35,7 +35,7 @@ $WORKDIR = catdir($FindBin::Bin, "logs");
$reslog = catfile($FindBin::Bin, updir, "blib", "script", "reslog");
$tno = 0;
# 1-6: Trim suffix is the same as suffix
# 1-8: Trim suffix is the same as suffix
foreach my $st (@SOURCE_TYPES) {
# 1: Trim suffix is the same as suffix
$_ = eval {
@ -126,7 +126,7 @@ foreach my $st (@SOURCE_TYPES) {
clean_up $_ || $$st{"skip"}, $WORKDIR, ++$tno;
}
# 7: A same log file is specified more than once
# 9: A same log file is specified more than once
$_ = eval {
my ($title, $cmd, $ret_no, $out, $err, %logfiles);
my ($fr, $frb, @fle, $fle, $flr, %cef, %crf, %tef, %trf);

74
t/_helper.pm
View File

@ -1,6 +1,6 @@
# _helper.pm - A simple test suite helper
# Copyright (c) 2005-2021 imacat
# Copyright (c) 2005-2022 imacat
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -25,10 +25,10 @@ $VERSION = "0.05";
@EXPORT = qw(
read_file read_raw_file write_file write_raw_file
run_cmd where_is file_type list_files preserve_source clean_up
has_no_file has_no_gzip has_no_bzip2
has_no_file has_no_gzip has_no_bzip2 has_no_xz
make_log_file make_resolved_log_file make_empty_log_file
random_word
TYPE_TEXT TYPE_GZIP TYPE_BZIP2
TYPE_TEXT TYPE_GZIP TYPE_BZIP2 TYPE_XZ
@CONTENT_TYPES @SOURCE_TYPES @KEEP_MODES @OVERRIDE_MODES @SUFFICES @TRIM_SUFFIX);
# Prototype declaration
sub this_file();
@ -45,6 +45,7 @@ sub clean_up($$$);
sub has_no_file();
sub has_no_gzip();
sub has_no_bzip2();
sub has_no_xz();
sub make_log_file($);
sub make_resolved_log_file($);
sub make_empty_log_file($);
@ -60,16 +61,18 @@ use File::Spec::Functions qw(splitdir catdir catfile path);
use File::Temp qw(tempfile);
use Socket;
our (%WHERE_IS, $HAS_NO_FILE, $HAS_NO_GZIP, $HAS_NO_BZIP2, $RANDOM_IP);
our (%WHERE_IS, $HAS_NO_FILE, $HAS_NO_GZIP, $HAS_NO_BZIP2, $HAS_NO_XZ, $RANDOM_IP);
%WHERE_IS = qw();
undef $HAS_NO_FILE;
undef $HAS_NO_GZIP;
undef $HAS_NO_BZIP2;
undef $HAS_NO_XZ;
undef $RANDOM_IP;
use constant TYPE_TEXT => "text/plain";
use constant TYPE_GZIP => "application/x-gzip";
use constant TYPE_BZIP2 => "application/x-bzip2";
use constant TYPE_XZ => "application/x-xz";
our (@CONTENT_TYPES, @SOURCE_TYPES, @KEEP_MODES, @OVERRIDE_MODES, @SUFFICES,
@TRIM_SUFFIX);
@ -94,7 +97,11 @@ our (@CONTENT_TYPES, @SOURCE_TYPES, @KEEP_MODES, @OVERRIDE_MODES, @SUFFICES,
{ "title" => "bzip2 source",
"type" => TYPE_BZIP2,
"suf" => ".bz2",
"skip" => has_no_bzip2, }, );
"skip" => has_no_bzip2, },
{ "title" => "xz source",
"type" => TYPE_XZ,
"suf" => ".xz",
"skip" => has_no_xz, }, );
# All the keep mode information
@KEEP_MODES = (
{ "title" => "keep default",
@ -239,6 +246,29 @@ sub read_file($) {
return $content;
}
# an xz compressed file
} elsif ($file =~ /\.xz$/) {
# IO::Uncompress::UnXz
if (eval { require IO::Uncompress::UnXz; 1; }) {
my $xz;
$content = "";
$xz = IO::Uncompress::UnXz->new($file)
or die this_file . ": $file: $IO::Uncompress::UnXz::UnXzError";
$content = join "", <$xz>;
$xz->close or die this_file . ": $file: $IO::Uncompress::UnXz::UnXzError";
return $content;
# xz executable
} else {
my ($PH, $CMD);
$CMD = where_is "xz";
$CMD = "\"$CMD\" -cdf \"$file\"";
open $PH, "$CMD |" or die this_file . ": $CMD: $!";
$content = join "", <$PH>;
close $PH or die this_file . ": $CMD: $!";
return $content;
}
# a plain text file
} else {
my $FH;
@ -326,6 +356,29 @@ sub write_file($$) {
return;
}
# an xz compressed file
} elsif ($file =~ /\.xz$/) {
# IO::Compress::Xz
if (eval { require IO::Compress::Xz; 1; }) {
my $xz;
$xz = IO::Compress::Xz->new($file, Extreme => 1)
or die this_file . ": $file: $IO::Compress::Xz::XzError";
($xz->write($content) == length $content)
or die this_file . ": $file: $IO::Compress::Xz::XzError";
$xz->close or die this_file . ": $file: $IO::Compress::Xz::XzError";
return;
# xz executable
} else {
my ($PH, $CMD);
$CMD = where_is "xz";
$CMD = "\"$CMD\" -c9f > \"$file\"";
open $PH, "| $CMD" or die this_file . ": $CMD: $!";
print $PH $content or die this_file . ": $CMD: $!";
close $PH or die this_file . ": $CMD: $!";
return;
}
# a plain text file
} else {
my $FH;
@ -418,6 +471,7 @@ sub file_type($) {
$_ = File::MMagic->new->checktype_filename($file);
return TYPE_GZIP if /gzip/;
return TYPE_BZIP2 if /bzip2/;
return TYPE_XZ if /xz/;
# All else are text/plain
return TYPE_TEXT;
}
@ -426,6 +480,7 @@ sub file_type($) {
$_ = join "", `"$_" "$file"`;
return TYPE_GZIP if /gzip/;
return TYPE_BZIP2 if /bzip2/;
return TYPE_XZ if /: XZ/;
# All else are text/plain
return TYPE_TEXT;
}
@ -515,6 +570,15 @@ sub has_no_bzip2() {
return $HAS_NO_BZIP2;
}
# If we have xz support somewhere
sub has_no_xz() {
$HAS_NO_XZ = eval { require IO::Compress::Xz; require IO::Uncompress::UnXz; 1; }
|| defined where_is "xz"?
0: "IO::Compress::Xz or xz executable not available"
if !defined $HAS_NO_XZ;
return $HAS_NO_XZ;
}
# Create a normal random log file
sub make_log_file($) {
local ($_, %_);