mflu / collective-intelligence-framework Goto Github PK

::Botnet
::Infrastructure

etc.

use plugins if appropriate.

Feature Description:

Target Audience:

Purpose:

cif_feed_parser -c /etc/cif/misc.cfg -f malc0de | grep vlc
<description>URL: www.telecharger-v.lc/download.php?file=vlc<=en, IP Address: 
188.165.234.84, Country: FR, ASN: 16276, MD5: 
280fdf7021d8eff472600a29ae489ccf</description>
Died at 
/home/wes/projects/src/cif/server/CIF-FeedParser/lib/CIF/FeedParser/ParseRss.pm 
line 19.
$ cif_feed_parser -c /etc/cif/misc.cfg -f malc0de | grep vlc

not well-formed (invalid token) at line 158, column 65, byte 9067 at 
/usr/lib/perl5/XML/Parser.pm line 187

Feature Description:

Target Audience:

Purpose:

Feature Description:

Target Audience:

+ chrome
+ firefox
+ mail.app

Purpose:

http://code.google.com/p/collective-intelligence-framework/source/browse/#svn%2F
trunk%2Fserver%2FCIF-Archive-Storage-Plugin-Vcard

http://www.infiltrated.net/vabl.txt

http://code.google.com/p/collective-intelligence-framework/source/browse/#svn%2F
trunk%2Fserver%2FCIF-Archive-DataType-Plugin-Related

http://danger.rulez.sk/projects/bruteforceblocker/blist.php

{{{
// ==UserScript==
// @name        Portal Linkify Plus
// @version     2.0.1
// @namespace   http://arantius.com/misc/greasemonkey/
// @description Turn plain text URLs into links.  Supports http, https, ftp, 
email addresses
// @require     
http://arantius.com/misc/greasemonkey/imports/content-scope-runner.user.js
// @include     *
// @exclude     http://www.google.tld/search*
// ==/UserScript==

/*******************************************************************************
Loosely based on the Linkify script located at:
  http://downloads.mozdev.org/greasemonkey/linkify.user.js

Originally written by Anthony Lieuallen of http://arantius.com/
Licensed for unlimited modification and redistribution as long as
this notice is kept intact.

If possible, please contact me regarding new features, bugfixes
or changes that I could integrate into the existing code instead of
creating a different script.  Thank you

Version history:
 Version 2.0.1:
  - Fix aberrant 'mailto:' where it does not belong.
 Version 2.0:
  - Apply incrementally, so the browser does not hang on large pages.
  - Continually apply to new content added to the page (i.e. AJAX).
 Version 1.1.4:
  - Basic "don't screw up xml pretty printing" exception case
 Version 1.1.3:
  - Include "+" in the username of email addresses.
 Version 1.1.2:
  - Include "." in the username of email addresses.
 Version 1.1:
  - Fixed a big that caused the first link in a piece of text to
    be skipped (i.e. not linkified).
*******************************************************************************/

var notInTags=[
    'a', 'head', 'noscript', 'option', 'script', 'style', 'title', 'textarea'
];
var textNodeXpath=
    ".//text()[not(ancestor::"+notInTags.join(') and not(ancestor::')+")]";
var 
ipRE=/(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-
9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-
9]?)/g;


var queue=[];

/******************************************************************************/

if ('text/xml'!=document.contentType
    && 'application/xml'!=document.contentType
) {
    linkifyContainer(document.body);
    document.body.addEventListener('DOMNodeInserted', function(event) {
        linkifyContainer(event.target);
    }, false);
}

/******************************************************************************/

function linkifyContainer(container) {
    var xpathResult=document.evaluate(
        textNodeXpath, container, null,
        XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, null
    ); 

    var i=0;
    function continuation() {
        var node, limit=0;
        while (node=xpathResult.snapshotItem(i++)) {
            linkifyTextNode(node);

            if (++limit>50) {
                return setTimeout(continuation, 0);
            }
        }
    }
    setTimeout(continuation, 0);
}

function linkifyTextNode(node) {
    var i, l, m;
    var txt=node.textContent;
    var span=null;
    var p=0;
    while (m=ipRE.exec(txt)) {
        if (null==span) {
            //create a span to hold the new text with links in it
            span=document.createElement('span');
        }

        //get the link without trailing dots
        l=m[0].replace(/\.*$/, '');
        //put in text up to the link
        span.appendChild(document.createTextNode(txt.substring(p, m.index)));
        //create a link and put it in the span
        a=document.createElement('a');
        a.className='linkifyplus';
        a.appendChild(document.createTextNode(l));
        l = "https://example.com/info?q=" + l
        a.setAttribute('href', l);
        span.appendChild(a);
        //track insertion point
        p=m.index+m[0].length;
    }
    if (span) {
        //take the text after the last link
        span.appendChild(document.createTextNode(txt.substring(p, txt.length)));
        //replace the original text with the new span
        try {
            node.parentNode.replaceChild(span, node);
        } catch (e) {
            console.error(e);
            console.log(node);
        }
    }
}
}}}

add functionality to ->convert to take in an array of rdata

hook into the prepare function for resolution

$ cif -q 1.1.1.1
Query: 1.1.1.1‬
Feed Restriction: private
Feed Created: 2011-01-24T02:37:50Z

restriction|severity|address  |portlist|detecttime            |description     
|alternativeid_restriction|alternativeid
private    |        |1.2.3.4/0|        |2011-01-21 16:00:00+00|search 
1.2.3.4/0|private                  |             

os x

have the client provide an option for searching rdata

have the client provide a summary by default, add option to provide details 
(eg: monthly, yearly aggregation).

fix required perms for chrome xtn

Query: domain/fastflux
Traceback (most recent call last):
  File "/usr/local/bin/cifcli.py", line 5, in <module>
    pkg_resources.run_script('cif==0.00-05', 'cifcli.py')
  File "/System/Library/Frameworks/Python.framework/Versions/2.6/Extras/lib/python/pkg_resources.py", line 442, in run_script
    self.require(requires)[0].run_script(script_name, ns)
  File "/System/Library/Frameworks/Python.framework/Versions/2.6/Extras/lib/python/pkg_resources.py", line 1160, in run_script
    execfile(script_filename, namespace, namespace)
  File "/Library/Python/2.6/site-packages/cif-0.00_05-py2.6.egg/EGG-INFO/scripts/cifcli.py", line 32, in <module>
    text = rclient.table(rclient.responseContent)
  File "/Library/Python/2.6/site-packages/cif-0.00_05-py2.6.egg/cif/__init__.py", line 109, in table
    t.add_row([item[col] or '' for col in cols])
KeyError: 'portlist'


UnicodeEncodeError: 'ascii' codec can't encode character u'\xa0' in position 
25: ordinal not in range(128)

url/malware -s low

Feature Description: implement http://code.google.com/p/apache2rest interface 
in addition to RT-CIF

Target Audience: everyone

Purpose: provide a generic mod_perl REST interface for people not using RT

python client should accept -c for multiple configs

Feature Description:

[5/5/11 9:18:35 AM] Wes: 
http://search.cpan.org/~santeri/VT-API-0.12/lib/VT/API.pm
[5/5/11 9:18:40 AM] Wes: http://www.virustotal.com/advanced.html#publicapi

Target Audience:

Purpose:

submit malware from the repo and get back VT analytics

+ possibly in the config as an option
+ figure out plugin options

CREATE INDEX for sql scripts

Feature Description:

Target Audience:

Purpose:

create datatype that will index and search blobs for country code

+ finish regex
+ commit to Regexp::Common
+ CIF::Archive::DataType::Plugin::Infrastructure integration

{{{
const ipv6_8hex_regex = /([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}/;
const ipv6_compressed_hex_regex = 
/(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4}
)*)?)/;
const ipv6_hex4dec_regex = 
/(([0-9A-Fa-f]{1,4}:){6,6})([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)/;
const ipv6_compressed_hex4dec_regex = 
/(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::(([0-9A-Fa-f]{1,4}:)*)([0-9]+)\.([0-
9]+)\.([0-9]+)\.([0-9]+)/;
}}}

http://code.google.com/p/kippo/

possibly make "my domain" by default?

create tools to archive the messages table

CIF::Archive->insert() should support multiple data-type plugins

incorporate these checks into the feed-inserts (boost or lower the 
severity/confidence)

see perl plugins

+ infrastructure/network feed
+ infrastructure/asn feed

Feature Description:

Target Audience:

Purpose:

same as we do with the ASN's.

break out of DomainSimple.pm -- make it it's own analytic

Feature Description:

Target Audience:

Purpose:

http://en.wikipedia.org/wiki/Dynamic_DNS
http://www.dmoz.org/Computers/Internet/Protocols/DNS/DNS_Providers/Dynamic_DNS/
http://directory.google.com/alpha/Top/Computers/Software/Internet/Servers/Addres
s_Management/
http://www.changeip.com/freedomains.asp
http://www.malwaredomains.com/files/dynamic_dns.txt

+ installation doc/videos
+ overview doc/videos

Clean stuff up

Feature Description:

Target Audience:

Purpose:

+ create generic library
+ auto-detect RSS vs plaintext
+ take in regex's
+ incorporate 3rd party checks
+ leverage WebAPI (?) where possible

 # clean it up
 # make more "generic" to be in-sync with CIF::Archive

streamline scripts so they use the nameservers in ~/.cif OR resolve.conf

eg: http://search.cpan.org/~tmtm/Class-DBI-v3.0.17/lib/Class/DBI/Test/SQLite.pm

What steps will reproduce the problem?
1.
2.
3.

What is the expected output? What do you see instead?


Please use labels and text to provide additional information.

make overwrite-able from server side config.

client.GET should return the array of query/feed data

search box should be along side text

provide "silent" flag in the client

# messages / disk space used
# time vs space
# mail report monthly with projection

Feature Description:
http://www.csiir.ornl.gov/shue/research/infocommini10.pdf
http://www.terena.org/activities/tf-csirt/meeting32/dulaunoy-bgpranking.pdf
https://www.cs.indiana.edu/~minaxi/pubs/ceas09.pdf
https://www.cs.indiana.edu/~minaxi/pubs/hicss10.pdf

Target Audience:

Purpose: