X-Git-Url: https://jxself.org/git/?a=blobdiff_plain;f=deblob-check;h=78136262d55591b2f750d72fb9d8ada6c28c4e95;hb=03b05fb6af7472b4df877f4a01eb91abd1c89e0d;hp=f376cce0143a0ebe89df5e6ea3954465e7073009;hpb=50e7fbbb480d8200e2777512f5179cb99bba3b2d;p=releases.git diff --git a/deblob-check b/deblob-check index f376cce0143a..78136262d555 100755 --- a/deblob-check +++ b/deblob-check @@ -1,13 +1,13 @@ #! /bin/sh -# deblob-check version 2008-12-11 +# deblob-check version 2009-01-24 # Inspired in gNewSense's find-firmware script. # Written by Alexandre Oliva # Check http://www.fsfla.org/svn/fsfla/software/linux-libre for newer # versions. -# Copyright (C) 2008 Alexandre Oliva +# Copyright (C) 2008, 2009 Alexandre Oliva # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,10 +26,11 @@ # usage: deblob-check [-S] [-vv] [-s S] [-lDdBbCcXxPpFftVh?H] \ -# [*.tar* patch-* *.patch *.diff] +# *.tar* patch-* [-i prefix/] *.patch *.diff... -# Look for too-long undocumented sequences of numbers (generally blobs -# in disguise) in source files. +# Look for and report too-long undocumented sequences of numbers +# (generally blobs in disguise) in source files, as well as requests +# for loading non-Free firmware. # The order of command line flags is significant. Flags given out of # the order above won't be handled correctly, sorry. @@ -109,6 +110,27 @@ # -v --verbose: increase verbosity level, for internal debugging. May # be given at most twice. + +# file options: + +# --: Don't process command-line options any further. All following +# arguments are taken as filenames. + +# -i --implied-prefix --prefix: prepend the given prefix to each filename +# listed after this option, when configuring false positives +# and negatives. + +# *.tar*: iterate over all files in the named tar file. + +# *.patch, patch-*, *.diff: Look for blobs in the [ +] parts of the +# *patch, unless --reverse-patch is given, in which case +# the [ -] parts will be used. + +# Anything else is assumed to be a source file. + +# *.gz | *.bz2: Decompress automatically. + + # The exit status is only significant for the --list options: it will # be true if nothing was found, and false otherwise. @@ -261,6 +283,20 @@ case $1 in ;; esac +prefix=/ +case $1 in +--implied-prefix | --prefix| -i) + prefix=$2 + case $prefix in + /*/) ;; + */) prefix=/$prefix ;; + /*) prefix=$prefix/ ;; + *) prefix=/$prefix/ ;; + esac + shift 2 || exit 1 + ;; +esac + test_mode=false name=deblob-check @@ -386,9 +422,10 @@ q 1;" *) case $1 in - -- | -l | --list-blobs) shift;; + -l | --list-blobs) shift;; esac case $1 in + -- | --implied-prefix | --prefix | -i) ;; -*) if test ! -f "$1"; then echo "$name: \`$1' given too late or out of the proper sequence." >&2 @@ -409,7 +446,9 @@ q 1;" esac case $1 in ---) shift;; +--) + sawdashdash=t + shift;; esac if $test_mode; then @@ -520,30 +559,23 @@ set_except () { addx "$1\\([^\\n]*\\\\[\\n]\\)*[^\\n\\\\]*$eol" $2 } - # Match $1 in $2 as a blob. Not implemented yet. - blob () { - : + # Match $1 in $2 as a blob. Not anchored. + blobna () { + badx "$1" $2 } - case /$1 in - /drivers/net/tg3.c) - # This file contains firmwares that we deblob with high - # sensitivity, so make sure the sequences of numbers that are not - # blobs are not deblobbed. FIXME: we should have patterns to - # recognize the blobs instead. - defsnc ' static const u32 test_pat\[4\]\[6\] =' drivers/net/tg3.c - accept " }\\(,\\? mem_tbl_5\\(70x\\|705\\|755\\|906\\)\\[\\] = {$sepx$blobpat*$sepx}\\)*;" drivers/net/tg3.c - ;; + # Match $1 in $2 as a blob. The expectation is a match in the + # beginning of line, but we don't do anchoring of blob patterns ATM. + blob () { + badx "$1" $2 + } - /drivers/media/video/gspca/conex.c) - # FIXME: we shouldn't have to duplicate this here, but deblob - # doesn't pass us a pathname that matches linux-*.*.*/, and then - # we end up deblobbing false positives. - defsnc 'static const __u8 cx_inits_\(176\|320\|352\|640\)\[\] =' drivers/media/video/gspca/conex.c - defsnc 'static const __u8 cx_jpeg_init\[\]\[8\] =' drivers/media/video/gspca/conex.c - defsnc 'static const __u8 cxjpeg_\(640\|352\|320\|176\|qtable\)\[\]\[8\] =' drivers/media/video/gspca/conex.c - ;; + blobna "request_firmware_nowait" + blobna "request_firmware" + blobna "request_ihex_firmware" + blobna "MODULE_FIRMWARE([^;]*)[^;]*;" + case $prefix$1 in */*linux*.tar* | */*kernel*.tar* | */*linux-*.*.*/*) # false alarms, contain source # drivers/net/wan/wanxlfw.inc_shipped -> wanxlfw.S @@ -586,7 +618,7 @@ set_except () { accept 'string_product:[\n]\?\(;\? \.byte[^\n]*[\n]\)\+string_product_end:' 'drivers/usb/serial/\(keyspan_pda\|xircom_pgs\).S' accept ' [/][*] \(SQCIF\|QSIF\|QCIF\|SIF\|CIF\|VGA\) [*][/][\n] {[\n] {'"$blobpat*" drivers/media/video/pwc/pwc-nala.h accept 'P[13]\([\n]#[^\n]*\)*[\n]*\([\n][0-9 ]*\)\+' drivers/video/logo/*.ppm - defsnc 'for i in [ 0-9\\\n]*[\n]do' Documentation/specialix.txt + accept 'for i in [ 0-9\\\n]*[\n]do' 'Documentation/specialix\.txt|Documentation/serial/specialix\.txt' accept ' : 3600000 3400000 3200000 3000000 2800000 ' Documentation/cpu-freq/cpufreq-stats.txt accept '00 00[\n]64 01[\n]8e 0b[\n][\n][0-9a-f \n]*fe fe' 'Documentation/scsi/\(sym\|ncr\)53c8xx_2.txt' accept '0f 00 08 08 64 00 0a 00 - id 0[\n]'"$blobpat*" 'Documentation/scsi/\(sym\|ncr\)53c8xx_2.txt' @@ -618,6 +650,13 @@ set_except () { defsnc 'static const char zr360[56]0_dqt\[0x86\] =' 'drivers/media/video/zr36060\.c\|drivers/media/video/zoran/zr36060\.c' defsnc 'static u8 tas3004_treble_table\[\] =' sound/aoa/codecs/tas-basstreble.h + # This file contains firmwares that we deblob with high + # sensitivity, so make sure the sequences of numbers that are not + # blobs are not deblobbed. FIXME: we should have patterns to + # recognize the blobs instead. + defsnc ' static const u32 test_pat\[4\]\[6\] =' drivers/net/tg3.c + accept " }\\(,\\? mem_tbl_5\\(70x\\|705\\|755\\|906\\)\\[\\] = {$sepx$blobpat*$sepx}\\)*;" drivers/net/tg3.c + # end of generic checked expressions. # version-specific checked bits start here @@ -752,6 +791,78 @@ set_except () { defsnc ';[/][*]@@ -[0-9]*,[0-9]* +[0-9]*,[0-9]* @@ static const __u8 \(hv7131r\|mi0360\|mo4000\|ov76\(60\|48\)\)_sensor_init\[\]\[8\] = {[*][/];' drivers/media/video/gspca/sonixj.c defsnc 'static \(const \)\?u32 ar\(5416\|9280\)\(Modes\(_fast_clock\)\?\|Common\|BB_RfGain\|Bank6\(TPC\)\?\|Addac\)\(_91[06]0\(1_1\)\?\|_9280\(_2\)\?\)\?\[\]\[[236]\] =' drivers/net/wireless/ath9k/initvals.h + # new in 2.6.28 + accept '\(static \)\?const char \(inv\)\?parity\[256\] = {[ \n01,]*};' 'Documentation/mtd/nand_ecc\.txt\|drivers/mtd/nand/nand_ecc\.c' + defsnc 'static const char \(bitsperbyte\|addressbits\)\[256\] =' drivers/mtd/nand/nand_ecc.c + defsnc 'static struct pinmux_cfg_reg pinmux_config_regs\[\] =' arch/sh/kernel/cpu/sh2a/pinmux-sh7203.c + defsnc ' static const u8 e_keymap\[\] =' drivers/hid/hid-lg.c + defsnc ' *struct phy_reg phy_reg_init\(_[01]\)\?\[\] =' drivers/net/r8169.c + defsnc 'DEFINE_DEFAULT_PDR(0x0161, 256,' drivers/net/wireless/hermes_dld.c + defsnc 'static const int isink_cur\[\] =' drivers/regulator/wm8350-regulator.c + defsnc 'static const s16 \(converge_speed_ipb\?\|LAMBDA_table\[4\]\)\[101\] =' drivers/staging/go7007/go7007-fw.c + defsnc 'static const u32 addrinctab\[33\]\[2\] =' drivers/staging/go7007/go7007-fw.c + defsnc 'static const u8 \(default_intra_quant_table\|\(val\|bits\)_[ad]c_\(lu\|chro\)minance\)\[\] =' drivers/staging/go7007/go7007-fw.c + defsnc 'static const int zz\[64\] =' drivers/staging/go7007/go7007-fw.c + defsnc ' u16 pack\[\] =' drivers/staging/go7007/go7007-fw.c + defsnc 'static u8 \(initial\|channel\)_registers\[\] =' 'drivers/staging/go7007/wis-\(ov7640\|saa7113\|tw2804\).c' + defsnc 'u16 MTO_One_Exchange_Time_Tbl_[ls]\[MTO_MAX_FRAG_TH_LEVELS\]\[MTO_MAX_DATA_RATE_LEVELS\] =' drivers/staging/winbond/mto.c + defsnc 'u32 \(al2230_txvga_data\|w89rf242_txvga_old_mapping\)\[\]\[2\] =' drivers/staging/winbond/reg.c + defsnc 'static const UINT16 crc16tab\[256\] =' drivers/staging/wlan-ng/hfa384x.c + defsnc 'static const UINT32 wep_crc32_table\[256\] =' drivers/staging/wlan-ng/p80211wep.c + defsnc 'static const unsigned char wm_vol\[256\] =' sound/pci/ice1712/phase.c + defsnc 'static const u16 wm8900_reg_defaults\[WM8900_MAXREG\] =' sound/soc/wm8900.c + defsnc '} \(clk_sys_ratios\|bclk_divs\)\[\] =' sound/soc/wm8903.c + defsnc 'static u8 af9015_ir_table_\(leadtek\|twinhan\|a_link\|msi\|mygictv\|kworld\)\[\] =' drivers/media/dvb/dvb-usb/af9015.h + defsnc 'static struct snr_table \(qpsk\|qam\(16\|64\)\)_snr_table\[\] =' drivers/media/dvb/frontends/af9013_priv.h + defsnc 'static struct regdesc \(ofsm_init\|tuner_init_\(env77h11d5\|mt2060\(\|_2\)\|mxl500\(3d\|5\)\|qt1010\|mc44s803\|unknown\|tda18271\)\)\[\] =' drivers/media/dvb/frontends/af9013_priv.h + defsnc 'static u8 stv0288_earda_inittab\[\] =' drivers/media/dvb/frontends/eds1547.h + defsnc 'static u8 serit_sp1511lhb_inittab\[\] =' drivers/media/dvb/frontends/si21xx.c + defsnc 'static u8 stv0288_inittab\[\] =' drivers/media/dvb/frontends/stv0288.c + defsnc 'static const struct rf_channel rf_vals_b\[\] =' drivers/net/wireless/rt2x00/rt2400pci.c + defsnc 'static const u8 rtl8187b_reg_table\[\]\[3\] =' drivers/net/wireless/rtl8187_dev.c + + # request_firmware matches for 2.6.28 + accept 'D: Firmware loader (request_firmware)' CREDITS + accept 'FIRMWARE LOADER (request_firmware)' MAINTAINERS + accept ' - request_firmware() hotplug interface info.' Documentation/00-INDEX + accept 'This driver requires a patch for firmware_class[^\n]*[\n]request_firmware_nowait function\.' Documentation/dell_rbu.txt + accept ' request_firmware() hotplug interface:[\n] --*[\n].* - request_firmware_nowait() is also provided for convenience' Documentation/firmware_class/README + accept 'Still, there are kernel threads that may want.*For example, if request_.*_firmware() will fail regardless' Documentation/power/freezing-of-tasks.txt + accept 'Also, there may be some operations,.*calling request_firmware() from their .resume() routines' Documentation/power/notifiers.txt + accept 'There is an USB interface for downloading[/]uploading.*request_firmware interface\.' Documentation/video4linux/si470x.txt + accept '[\t]- move firmware loading to request_firmware()' drivers/staging/slicoss/README + accept 'config FIRMWARE_IN_KERNEL.*let firmware be loaded from userspace\.' drivers/base/Kconfig + accept 'static int[\n]_request_firmware(const struct firmware \*\*firmware_p, const char \*name,[^{]*[\n]{[\n]\([^}]\|[^\n}]}*\)*[\n]}[\n]' drivers/base/firmware_class.c + accept 'static int[\n]request_firmware_work_func(void \*arg)[\n]{[\n]\([^}]\|[^\n}]}*\)*ret = _request_firmware(\([^}]\|[^\n}]}*\)*[\n]}[\n]' drivers/base/firmware_class.c + accept '[/][*][*][\n] [*] request_firmware: - send firmware [^{]*[\n]{[\n]\([^}]\|[^\n}]}*\)*[\n]}[\n]' drivers/base/firmware_class.c + accept '[/][*][*][\n] [*] request_firmware_nowait: asynchronous version[^{]*[\n]{[\n]\([^}]\|[^\n}]}*\)*[\n]}[\n]' drivers/base/firmware_class.c + accept 'EXPORT_SYMBOL(request_firmware\(_nowait\)\?);' drivers/base/firmware_class.c + accept 'int request_firmware\(_nowait\)\?([^;]*);' include/linux/firmware.h + accept 'static inline int request_firmware\(_nowait\)\?([^{]*)[\n]{[\n][\t]return -EINVAL;[\n]}[\n]' include/linux/firmware.h + accept 'static inline int \(maybe_\)\?reject_firmware\(_nowait\)\?([^{;]*)[\n]{[\n]\([^}]\|[^\n}]}*\)*[\n]}[\n]' include/linux/firmware.h + + accept 'static inline int request_ihex_firmware\?([^{]*)[\n]{[\n]\([^}]\|[^\n}]}*\)*[\n]}[\n]' include/linux/ihex.h + ocomment '[/][*] Optional firmware\([^\n]*[\n] [*]\)*[^\n]* MODULE_FIRMWARE()' + oprepline '#define MODULE_FIRMWARE(_firmware)' include/linux/module.h + accept ' [*] Sample code on how to use request_firmware() from drivers\.' samples/firmware_class/firmware_sample_driver.c + accept '[\t]\(retval\|error\) = request_firmware\(_nowait\)\?([^;]*"sample_driver_fw",[^;]*);' samples/firmware_class/firmware_sample_driver.c + ocomment ' [/][*] request_firmware blocks until userspace finished' samples/firmware_class/firmware_sample_driver.c + accept ' [ \t]*" request_firmware_nowait failed' samples/firmware_class/firmware_sample_driver.c + + # Hunting down non-Free firmware-loading code and instructions. + + blob 'atmsar11\.fw' drivers/atm/ambassador.c + + blob ' sprintf([^;]*fore200[^;]*FW_EXT[^;]*);' drivers/atm/fore200e.c + blobna '\(pc\|sb\)a200e\(_ecd\)\?\.bin[12]?' drivers/atm/fore200e.c + blobna 'The supplied firmware images.*fore.*Rebuild and reinstall[^.]*\.' Documentation/networking/fore200e.txt + + blobna 'intelliport2\.bin' drivers/char/ip2/ip2main.c + + blobna '\(agere\|prism\)_\(sta\|ap\)_fw\.bin' drivers/net/wireless/orinoco.c + blobna 'symbol_sp24t_\(prim\|sec\)_fw' drivers/net/wireless/orinoco.c + + # This looks suspicious, but it pretty much just sets stuff to zero. initnc 'static __u8 mode8420\(pro\|con\)\[\] =' drivers/media/video/cs8420.h @@ -981,8 +1092,8 @@ set_except () { initnc 'static int fifo_map\[\]\[MAX_TX_FIFOS\] =' initnc 'static int initial_lfsr\[\] =' initnc 'static int log_tbl\[129\] =' - initnc 'static int miro_fmtuner\[\] =' - initnc 'static int miro_tunermap\[\] =' + initnc 'static int miro_fmtuner\[\] =' drivers/media/video/bt8xx/bt-cards.c + initnc 'static int miro_tunermap\[\] =' drivers/media/video/bt8xx/bt-cards.c initnc 'static int register_size\[\] =' initnc 'static int reserve_list\[MAX_RES_ARGS\] =' initnc 'static int reverse6\[64\] =' @@ -1017,7 +1128,7 @@ set_except () { initnc 'static u8 bnx2_5706_stats_len_arr\[BNX2_NUM_STATS\] =' initnc 'static u8 bnx2_5708_stats_len_arr\[BNX2_NUM_STATS\] =' initnc 'static u8 flit_desc_map\[\] =' - initnc 'static u8 init_tab \[\] =' + defsnc 'static u8 init_tab \?\[\] =' 'drivers/media/dvb/frontends/cx2270\(0\|2\)\.c' initnc 'static u8 mac_reader\[\] =' initnc 'static u8 mt2131_config1\[\] =' drivers/media/dvb/frontends/mt2131.c # <= 2.6.25 initnc 'static u8 mt2131_config1\[\] =' drivers/media/common/tuners/mt2131.c # >= 2.6.26 @@ -1178,7 +1289,7 @@ set_except () { blob 'static PHY_UCODE PhyUcode\[\] =[^;]*;' drivers/staging/sxg/sxgphycode.h # CONFIG_SXG # ok from earlier releases - accept 'for i in [ 0-9\\\n]*[\n]do' Documentation/specialix.txt + accept 'for i in [ 0-9\\\n]*[\n]do' 'Documentation/specialix.txt|Documentation/serial/specialix.txt' defsnc 'static yyconst flex_int\(16\|32\)_t yy_[^[]*\[[0-9]*\] =' '.*\.lex\.c_shipped' defsnc 'static const yytype_u\?int\(8\|16\) yy[^\n []*\[\] =' '.*\.lex\.c_shipped' initnc ';[/][*]@@ -[0-9]*,[0-9]* +[0-9]*,[0-9]* @@ static const yytype_u\?int\(8\|16\) yy[^\n []*\[\] =[*][/];' '.*\.tab\.c_shipped' @@ -1329,7 +1440,7 @@ set_except () { initnc 'static u32 reg_init_initialize\[\] =' drivers/media/video/saa717x.c initnc ' } vals\[\] =' drivers/media/video/saa717x.c initnc 'static const u32 \(main\|gear\)_seedset\[BACKOFF_SEEDSET_ROWS\]\[BACKOFF_SEEDSET_LFSRS\] =' drivers/net/forcedeth.c - blob '^unsigned char \(IDX_ACTIVATE_\(READ\|WRITE\)\|\(CM\|ULP\)_\(ENABLE\|SETUP\)\|DM_ACT\) = '"$sepx$blobseq*$sepx;" drivers/s390/net/qeth_core_mpc.c # from drivers/s390/net/qeth_mpc.c in 2.6.25 + blob 'unsigned char \(IDX_ACTIVATE_\(READ\|WRITE\)\|\(CM\|ULP\)_\(ENABLE\|SETUP\)\|DM_ACT\) = '"$sepx$blobseq*$sepx;" drivers/s390/net/qeth_core_mpc.c # from drivers/s390/net/qeth_mpc.c in 2.6.25 initnc '} pll_table\[\] =' drivers/video/geode/lxfb_ops.c accept " { 0x00014284, 19688 },[\n] { 0x00011104, 20400 },[\n] { $blobpat* }," drivers/video/geode/lxfb_ops.c # won't be necessary in rc3 initnc 'static const u16 wm9713_reg\[\] =' sound/soc/codecs/wm9713.c @@ -1518,13 +1629,6 @@ defineblob='[ ]*#[ ]*define[ ]\+\([^\n]*\\[\n]\)*[^\n]*'"$blobseq"'\([^\n]*\\ # without any intervening label. asmblob="[a-zA-Z_.][^\\n:;#/ ]*[ ]*:\\([^:{}]*\\|$asmcomment\\)*$blobseq\\([^:]*\\|$asmcomment\\)*" -# Regular expression that matches one or more blobs without -# intervening line breaks. -sblobctx="\\([^\\n]*$blobfseq\\)\\+" - -# Regular expression that matches the context for a long blob match. -lblobctx="\\($initblob\\|$defineblob\\|$asmblob\\|$sblobctx\\)" - # Set up the sed script that will go through the (processed) input, # looking for sequences of blobs and printing whatever was requested. # It accepts 3 arguments. @@ -1538,7 +1642,26 @@ lblobctx="\\($initblob\\|$defineblob\\|$asmblob\\|$sblobctx\\)" # $4 is the action for every complete input pattern. set_sedmain () { - falsepos=`sed 's,^\\\|,,;s,^,\\\\(,;s,$,\\\\),' < "$falsepos_name"` + falsepos=`sed 's,^\\\|,,;s,^.,\\\\(&,;s,.$,&\\\\),' < "$falsepos_name"` + orfalseneg=`cat < "$falseneg_name"` + + case $orfalseneg in + "") + blobfast=$blobseq + bloblong=$blobfseq + ;; + *) + blobfast="\\($blobseq$orfalseneg\\)" + bloblong="\\($blobfseq$orfalseneg\\)" + ;; + esac + + # Regular expression that matches one or more blobs without + # intervening line breaks. + sblobctx="\\([^\\n]*$bloblong\\)\\+" + + # Regular expression that matches the context for a long blob match. + lblobctx="\\($initblob\\|$defineblob\\|$asmblob\\|$sblobctx\\)" if test -s "$falsepos_name"; then check_false_positives="$v:???falsepos @@ -1547,8 +1670,8 @@ $v:+++falsepos h; s/$bol$falsepos/\\1;\/**\/;/g; # See if, after removing all matches, we end up without any blobs. -$v:???blobseq -/$blobseq/!{ +$v:???blobfast +/$blobfast/!{ g; b falsepos; } @@ -1585,8 +1708,8 @@ g; $4 s/^\(;[/][*]begin [^\n]*[\n]\)*//; s/\\($bol;[/][*]\\(end [^\n]*\\)\\?[*][/];\\)*$//; -$v:???!blobseq -/$blobseq/!b clean; +$v:???!blobfast +/$blobfast/!b clean; $check_false_positives # Fall through. : blob @@ -1621,7 +1744,7 @@ $v:print_matches h; s/^\\($falsepos[^\\n]*\\)\\([\\n].*\\)\\?$/\\1/; $v:narrowed to match -/$blobfseq/ { +/$bloblong/ { i\\ ::: $file ::: p; @@ -1654,11 +1777,11 @@ $v:print_marked_matches h; s/^\\($falsepos[^\\n]*\\)\\([\\n].*\\)\\?$/\\1/; $v:narrowed to match -/$blobfseq/{ +/$bloblong/{ i\\ ::: $file ::: s/{\\($sepx\\)\\?$blobfseq\\($sepx\\)\\?}[ ]*;/{\/*(DEBLOBBED)*\/};/g; - s/$blobfseq/\/*(DEBLOBBED)*\//g; + s/$bloblong/\/*(DEBLOBBED)*\//g; p; } g; @@ -1675,10 +1798,11 @@ $v:print_blobs /^$falsepos/ { $v:delete false positive # This is tricky. We don't want to print the false positive. - /^$falsepos[^\\n]*$blobseq/ { + /^$falsepos[^\\n]*$blobfast/ { $v:delete false positive immediately followed by blob h; - s/^\\($falsepos\\)[^\\n]*$blobseq.*/\\1/; + s/^\\($falsepos\\).*/\\1/; + $v:matched false positive : print_blobs_match_loop /[\\n]/ { s/^[^\\n]*[\\n]//; @@ -1693,10 +1817,11 @@ $v:print_blobs s/^$falsepos//; b print_blobs_delete_to_eol; } -/^[^\\n]*$blobseq/! { +/^[^\\n]*$blobfast/! { $v:delete non-blob header h; - s/[\\n]\\($falsepos\\|[^\\n]*$blobseq\\).*//; + s/[\\n]\\($falsepos\\|[^\\n]*$blobfast\\).*//; + $v:matched non-blob header : print_blobs_nomatch_loop /[\\n]/ { s/^[^\\n]*[\\n]//; @@ -1711,11 +1836,11 @@ $v:print_blobs h; i\\ ::: $file ::: -s/^\\([^\\n]*\\($blobfseq[^\\n]*\\)\\+\\)\\([\\n].*\\)\\?$/\\1/; +s/^\\([^\\n]*\\($bloblong[^\\n]*\\)\\+\\)\\([\\n].*\\)\\?$/\\1/; $v:narrowed to blob p; g; -s/^\\([^\\n]*\\($blobfseq[^\\n]*\\)\\+\\)//; +s/^\\([^\\n]*\\($bloblong[^\\n]*\\)\\+\\)//; : print_blobs_delete_to_eol $v:delete to eol s/^[^\\n]*//; @@ -1728,10 +1853,11 @@ $v:print_marked_blobs /^$falsepos/ { $v:delete false positive # This is tricky. We don't want to print the false positive. - /^$falsepos[^\\n]*$blobseq/ { + /^$falsepos[^\\n]*$blobfast/ { $v:delete false positive immediately followed by blob h; - s/^\\($falsepos\\)[^\\n]*$blobseq.*/\\1/; + s/^\\($falsepos\\).*/\\1/; + $v:matched false positive : print_marked_blobs_match_loop /[\\n]/ { s/^[^\\n]*[\\n]//; @@ -1746,10 +1872,11 @@ $v:print_marked_blobs s/^falsepos//; b print_marked_blobs_delete_to_eol; } -/^[^\\n]*$blobseq/! { +/^[^\\n]*$blobfast/! { $v:delete non-blob header h; - s/[\\n]\\($falsepos\\|[^\\n]*$blobseq\\).*//; + s/[\\n]\\($falsepos\\|[^\\n]*$blobfast\\).*//; + $v:matched non-blob header : print_marked_blobs_nomatch_loop /[\\n]/ { s/^[^\\n]*[\\n]//; @@ -1764,13 +1891,13 @@ $v:print_marked_blobs h; i\\ ::: $file ::: -s/^\\([^\\n]*\\($blobfseq[^\\n]*\\)\\+\\)\\([\\n].*\\)\\?$/\\1/; +s/^\\([^\\n]*\\($bloblong[^\\n]*\\)\\+\\)\\([\\n].*\\)\\?$/\\1/; $v:narrowed to blob s/{\\($sepx\\)\\?$blobfseq\\($sepx\\)\\?}[ ]*;/{\/*(DEBLOBBED)*\/};/g; -s/$blobfseq/\/*(DEBLOBBED)*\//g; +s/$bloblong/\/*(DEBLOBBED)*\//g; p; g; -s/^\\([^\\n]*\\($blobfseq[^\\n]*\\)\\+\\)//; +s/^\\([^\\n]*\\($bloblong[^\\n]*\\)\\+\\)//; : print_marked_blobs_delete_to_eol $v:delete to eol s/^[^\\n]*//; @@ -1783,10 +1910,11 @@ $v:print_cblobs /^$falsepos/ { $v:delete false positive # This is tricky. We don't want to print the false positive. - /^$falsepos[^\\n]*$blobseq/ { + /^$falsepos[^\\n]*$blobfast/ { $v:delete false positive immediately followed by blob h; - s/^\\($falsepos\\)[^\\n]*$blobseq.*/\\1/; + s/^\\($falsepos\\).*/\\1/; + $v:matched false positive : print_cblobs_match_loop /[\\n]/ { s/^[^\\n]*[\\n]//; @@ -1805,6 +1933,7 @@ $v:print_cblobs $v:delete non-blob header h; s/[\\n]\\($falsepos\\|$lblobctx\\).*//; + $v:matched non-blob header : print_cblobs_nomatch_loop /[\\n]/ { s/^[^\\n]*[\\n]//; @@ -1819,11 +1948,11 @@ $v:print_cblobs h; i\\ ::: $file ::: -s/^\\($lblobctx[^\\n]*\\($blobfseq[^\\n]*\\)*\\)\\([\\n].*\\)\\?$/\\1/; +s/^\\($lblobctx[^\\n]*\\($bloblong[^\\n]*\\)*\\)\\([\\n].*\\)\\?$/\\1/; $v:narrowed to blob p; g; -s/^\\($lblobctx[^\\n]*\\($blobfseq[^\\n]*\\)*\\)//; +s/^\\($lblobctx[^\\n]*\\($bloblong[^\\n]*\\)*\\)//; : print_cblobs_delete_to_eol $v:delete to eol s/^[^\\n]*//; @@ -1836,10 +1965,11 @@ $v:print_marked_cblobs /^$falsepos/ { $v:delete false positive # This is tricky. We don't want to print the false positive. - /^$falsepos[^\\n]*$blobseq/ { + /^$falsepos[^\\n]*$blobfast/ { $v:delete false positive immediately followed by blob h; - s/^\\($falsepos\\)[^\\n]*$blobseq.*/\\1/; + s/^\\($falsepos\\).*/\\1/; + $v:matched false positive : print_marked_cblobs_match_loop /[\\n]/ { s/^[^\\n]*[\\n]//; @@ -1858,6 +1988,7 @@ $v:print_marked_cblobs $v:delete non-blob header h; s/[\\n]\\($falsepos\\|$lblobctx\\).*//; + $v:matched non-blob header : print_marked_cblobs_nomatch_loop /[\\n]/ { s/^[^\\n]*[\\n]//; @@ -1872,13 +2003,13 @@ $v:print_marked_cblobs h; i\\ ::: $file ::: -s/^\\($lblobctx[^\\n]*\\($blobfseq[^\\n]*\\)*\\)\\([\\n].*\\)\\?$/\\1/; +s/^\\($lblobctx[^\\n]*\\($bloblong[^\\n]*\\)*\\)\\([\\n].*\\)\\?$/\\1/; $v:narrowed to blob s/{\\($sepx\\)\\?$blobfseq\\($sepx\\)\\?}[ ]*;/{\/*(DEBLOBBED)*\/};/g; -s/$blobfseq/\/*(DEBLOBBED)*\//g; +s/$bloblong/\/*(DEBLOBBED)*\//g; p; g; -s/^\\($lblobctx[^\\n]*\\($blobfseq[^\\n]*\\)*\\)//; +s/^\\($lblobctx[^\\n]*\\($bloblong[^\\n]*\\)*\\)//; : print_marked_cblobs_delete_to_eol $v:delete to eol s/^[^\\n]*//; @@ -1888,10 +2019,11 @@ b print_marked_cblobs; : print_both $v:print_both -/^\\($falsepos\\|[^\\n]*$blobseq\\)/! { +/^\\($falsepos\\|[^\\n]*$blobfast\\)/! { $v:delete non-blob header h; - s/[\\n]\\($falsepos\\|[^\\n]*$blobseq\\).*//; + s/[\\n]\\($falsepos\\|[^\\n]*$blobfast\\).*//; + $v:matched non-blob header : print_both_nomatch_loop /[\\n]/ { s/^[^\\n]*[\\n]//; @@ -1906,11 +2038,11 @@ $v:print_both h; i\\ ::: $file ::: -s/^\\(\\($falsepos[^\\n]*\\|[^\\n]*$blobfseq[^\\n]*\\)\\($blobfseq[^\\n]*\\)*\\)\\([\\n].*\\)\\?$/\\1/; +s/^\\(\\($falsepos[^\\n]*\\|[^\\n]*$bloblong[^\\n]*\\)\\($bloblong[^\\n]*\\)*\\)\\([\\n].*\\)\\?$/\\1/; $v:narrowed to blob p; g; -s/^\\(\\($falsepos[^\\n]*\\|[^\\n]*$blobfseq[^\\n]*\\)\\($blobfseq[^\\n]*\\)*\\)//; +s/^\\(\\($falsepos[^\\n]*\\|[^\\n]*$bloblong[^\\n]*\\)\\($bloblong[^\\n]*\\)*\\)//; : print_both_delete_to_eol $v:delete to eol s/^[^\\n]*//; @@ -1939,9 +2071,9 @@ $v:list_matches h; s/^\\($falsepos[^\\n]*\\)\\([\\n].*\\)\\?$/\\1/; $v:narrowed to match -/$blobfseq/{ +/$bloblong/{ s/{\\($sepx\\)\\?$blobfseq\\($sepx\\)\\?}[ ]*;/{\/*(DEBLOBBED)*\/};/g; - s/$blobfseq/\/*(DEBLOBBED)*\//g; + s/$bloblong/\/*(DEBLOBBED)*\//g; } p; g; @@ -1958,10 +2090,10 @@ $v:list_blobs /^$falsepos/ { $v:print false positive # This is tricky. We don't want to deblob the false positive. - /^$falsepos[^\\n]*$blobseq/ { + /^$falsepos[^\\n]*$blobfast/ { $v:print false positive immediately followed by blob h; - s/^\\($falsepos\\)[^\\n]*$blobseq.*/\\1/; + s/^\\($falsepos\\)[^\\n]*$blobfast.*/\\1/; : list_blobs_match_loop /[\\n]/ { P; @@ -1981,10 +2113,10 @@ $v:list_blobs s/^\\($falsepos[^\\n]*\\)//; b list_blobs_delete_to_eol; } -/^[^\\n]*$blobseq/! { +/^[^\\n]*$blobfast/! { $v:print non-blob header h; - s/[\\n]\\($falsepos\\|[^\\n]*$blobseq\\).*//; + s/[\\n]\\($falsepos\\|[^\\n]*$blobfast\\).*//; p; : list_blobs_nomatch_loop /[\\n]/ { @@ -1998,13 +2130,13 @@ $v:list_blobs b list_blobs_delete_to_eol; } h; -s/^\\([^\\n]*\\($blobfseq[^\\n]*\\)\\+\\)\\([\\n].*\\)\\?$/\\1/; +s/^\\([^\\n]*\\($bloblong[^\\n]*\\)\\+\\)\\([\\n].*\\)\\?$/\\1/; $v:narrowed to blob s/{\\($sepx\\)\\?$blobfseq\\($sepx\\)\\?}[ ]*;/{\/*(DEBLOBBED)*\/};/g; -s/$blobfseq/\/*(DEBLOBBED)*\//g; +s/$bloblong/\/*(DEBLOBBED)*\//g; p; g; -s/^\\([^\\n]*\\($blobfseq[^\\n]*\\)\\+\\)//; +s/^\\([^\\n]*\\($bloblong[^\\n]*\\)\\+\\)//; : list_blobs_delete_to_eol $v:delete to eol s/^[^\\n]*//; @@ -2014,10 +2146,10 @@ b list_blobs; : list_both $v:list_both -/^\\($falsepos\\|[^\\n]*$blobseq\\)/! { +/^\\($falsepos\\|[^\\n]*$blobfast\\)/! { $v:print non-blob header h; - s/[\\n]\\($falsepos\\|[^\\n]*$blobseq\\).*//; + s/[\\n]\\($falsepos\\|[^\\n]*$blobfast\\).*//; p; : list_both_nomatch_loop /[\\n]/ { @@ -2031,13 +2163,13 @@ $v:list_both b list_both_delete_to_eol; } h; -s/^\\(\\($falsepos[^\\n]*\\|[^\\n]*$blobfseq[^\\n]*\\)\\($blobfseq[^\\n]*\\)*\\)\\([\\n].*\\)\\?$/\\1/; +s/^\\(\\($falsepos[^\\n]*\\|[^\\n]*$bloblong[^\\n]*\\)\\($bloblong[^\\n]*\\)*\\)\\([\\n].*\\)\\?$/\\1/; $v:narrowed to blob s/{\\($sepx\\)\\?$blobfseq\\($sepx\\)\\?}[ ]*;/{\/*(DEBLOBBED)*\/};/g; -s/$blobfseq/\/*(DEBLOBBED)*\//g; +s/$bloblong/\/*(DEBLOBBED)*\//g; p; g; -s/^\\(\\($falsepos[^\\n]*\\|[^\\n]*$blobfseq[^\\n]*\\)\\($blobfseq[^\\n]*\\)*\\)//; +s/^\\(\\($falsepos[^\\n]*\\|[^\\n]*$bloblong[^\\n]*\\)\\($bloblong[^\\n]*\\)*\\)//; : list_both_delete_to_eol $v:delete to eol s/^[^\\n]*//; @@ -2060,6 +2192,8 @@ check () { falsepos_name=`mktemp -t deblob-check-falsepos-XXXXXX` tempfiles="$falsepos_name" + falseneg_name=`mktemp -t deblob-check-falseneg-XXXXXX` + tempfiles="$tempfiles $falseneg_name" # Add $1 to falsepos. Its usage makes it implicitly anchored to the # beginning of the line. $2, if present, will some day narrow the @@ -2070,11 +2204,20 @@ check () { fi } + # Add $1 to falseneg. Unlike addx, it is NOT implicitly anchored to + # the beginning of the line. $2, if present, will some day narrow + # the falseneg matches to files that match it. + badx () { + if test -n "$1"; then + echo -n "\\|$1" >> $falseneg_name + fi + } + set_except "$input" set_sed_cmd "$input" - rm -f "$falsepos_name" + rm -f $tempfiles tempfiles= # Choose the input source... @@ -2186,9 +2329,37 @@ pass=: tempfiles= trap "status=$?; test -z \"$tempfiles\" || rm -f $tempfiles; (exit $status); exit" 0 1 2 15 +process_arg= + # Go through each of the input files in the command line. for file do + case $process_arg in + "") ;; + --implied-prefix | --prefix | -i) + prefix=$file + case $prefix in + /*/) ;; + */) prefix=/$prefix ;; + /*) prefix=$prefix/ ;; + *) prefix=/$prefix/ ;; + esac + process_arg= + continue + ;; + *) + echo Internal error with process_arg=$process_arg >&2 + exit 1 + ;; + esac + + case $sawdashdash$file in + --implied-prefix | --prefix | -i) + process_arg=$file + continue + ;; + esac + # If we print anything whatsoever (even a blank line) while # processing it, we've failed. if check "$file"; then @@ -2200,6 +2371,14 @@ do fi done +case $process_arg in +"") ;; +*) + echo Missing argument to $process_arg >&2 + exit 1 + ;; +esac + #list: shift $n #list: exec test $# = 0