Sunday, March 27, 2011

Advanced Internet Search

Advanced Internet Search or AIS
is an program that speeds up the search process
this program goes to Google and looks want you gave it to search
then collect a number of results (number may differ when using different settings), each result will be analyzed by the application, if it found anything useful then it will be displayed in Results (see screenshot)

AIS comes with an build in Console to see what is has done or is doing (can be turned off), contents of the console can be copied or saved.


Screenshot:
 Usage:
  Main Form:
       - In Internet Search objects you insert items (right click) that will be used to search with the search Engine
       - In Page Search objects you insert items (right click) that will be used to search in pages
       - Save Results
       - Process: Start or Stop
       - Settings
       - Results:  Result Count, Find unnecessary Items and remove them
  Settings:
       - Show Console
       - Number of Urls that will be collect for each Internet Search Object (200 is around the first 20 pages of google results)(multiply by 10)
  Console:
       - Save Content (right click)
       - Copy Content (right click)

Features:
- the whole process is now faster (not ± 20 min, but more 5 - 10 min for 200 Urls per Internet search object) then the older version
- better searching
- Save Results
- Settings included
- Results Handling
- Console display

Future Features:
- Chose which Search Engines will be used

AIS comes with an extra feature:
Filter list
this is a list of Items (separated by #) that will by replaced with nothing, in the Internet/Pages Search Objects
mostly useless words like: The
this list is still sort so any suggestions? (you can edit it yourself with notepad)

Need:
- Framework .net 2.0

this Program comes with a Curse though:
it reacts really bad to the stop command.
and it takes time to actually press the stop button
but if you have Patience you will reach it so don't freak out

Download

here is the code(VB)
Imports System.IO
Imports System.Text

Public Class MainDialog
    Public FoUr, Results, FilterOutList As New List(Of String)
    Public SearchEngineFilterlist, ListofSearchEngine As New List(Of String)
    Public UrlCount As Integer = 200
    Public ProcesStop As Boolean = False
    Public ShowConsole As Boolean = True
    Public SearchEngine As String = "Google"
    Public SearchEngineURL As String = ""
    'Setup
    Private Sub MainDialog_Load(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles MyBase.Load
        ListofSearchEngine = GetListOfSearchEngine()
        If ShowConsole = True Then Console.Show()
        ProText.Text = ""
        FilterOutList = GetFilterList()
        SearchEngineURL = GetSearchEngineURL(SearchEngine)
        SearchEngineFilterlist = GetSearchEngineFilterlist(SearchEngine)
        UpdateStuff()
    End Sub

    'Proces
    Private Sub StopToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles StopToolStripMenuItem.Click
        ProcesStop = True
        Application.DoEvents()
        StartToolStripMenuItem.Enabled = True
        ProText.Text = "Trying to kill Process"
        ResetStuff()
        Do Until ProcesStop = True
            ProcesStop = True
        Loop
    End Sub
    Private Sub StartToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles StartToolStripMenuItem.Click
        'Voorbereiding

        CheckSearchObjects()
        If ShowConsole = True Then Console.Show()
        If InternetSearchObjects.Items.Count = 0 Then Exit Sub
        StartToolStripMenuItem.Enabled = False
        SetProgressBar(InternetSearchObjects.Items.Count - 1, 0, 0)
        MenuStrip1.Update()
        UpdateStuff()
        'Process I
        For ItemCount As Integer = 0 To InternetSearchObjects.Items.Count - 1
            Dim Total As Integer = 0
            Dim Item As String = InternetSearchObjects.Items(ItemCount).ToString
            For PageNumber As Integer = 0 To Math.Round((UrlCount / 10), 0)
                UpdateStuff(ItemCount, "Getting Urls for: " & Item, "A.I.S.:" & Item)
                Dim Page As String = GetSearchEnginePage(Item, PageNumber)
                Dim Urls() As String = GetUrls(Page)
                Urls = FilterUrls(Urls)
                Total += UBound(Urls)

                If ProcesStop = True Then
                    ResetStuff()
                    Exit Sub
                End If

                For Index As Integer = 0 To UBound(Urls)
                    If Urls(Index) <> "" Then FoUr.Add(Urls(Index))
                    If ShowConsole = True And Urls(Index) <> "" Then Console.Writeline(Urls(Index))
                Next

                If Total >= UrlCount Then Exit For
            Next
        Next
        'Process II
        SetProgressBar(FoUr.Count, 0, 0)
        For UrlIndex As Integer = 0 To FoUr.Count - 1
            If ProcesStop = True Then
                ProcesStop = False
                ResetStuff()
                Exit Sub
            End If
            Dim Page As String = GetPage(FoUr(UrlIndex)).ToLower
            Page = Page.ToLower
            If ShowConsole = True Then Console.Writeline("Checking: " & FoUr(UrlIndex) & vbNewLine)
            UpdateStuff(UrlIndex, "Checking: " & FoUr(UrlIndex), "A.I.S.:" & FoUr(UrlIndex))
            Application.DoEvents()
            Dim TmpRes As New List(Of String)
            For I As Integer = 0 To PageSearchObject.Items.Count - 1
                If Page.Contains(PageSearchObject.Items(I).ToString.ToLower) Then FoundInformation.Items.Add(PageSearchObject.Items(I))
                TmpRes = GetString(PageSearchObject.Items(I).ToString.ToLower, Page)
                For J As Integer = 0 To TmpRes.Count - 1
                    If DetectUrl(TmpRes(J), PageSearchObject.Items(I)) = False Then
                        FoundInformation.Items.Add(TmpRes(J) & "@ " & FoUr(UrlIndex))
                        Results.Add(TmpRes(J) & "@ " & FoUr(UrlIndex))
                        If ShowConsole = True Then Console.Writeline(TmpRes(J))
                    End If
                    If ProcesStop = True Then
                        ResetStuff()
                        ProcesStop = False
                        Exit Sub
                    End If
                Next
            Next

        Next
        ResetStuff()
    End Sub

    'Handelingen
    '   InSeOb
    Private Sub InsertToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles InsertToolStripMenuItem.Click
        Dim Add As String = InputBox("Search Object to be used in Search Engine" & vbCrLf & "Seperate Items are Created when you use: ,", "Adding a Internet Search Object")
        If Add.Length >= 1 Then
            If Add.Contains(",") Then
                Dim split() As String = Add.Split(",")
                SetProgressBar(UBound(split), 0)
                For I As Integer = 0 To UBound(split)
                    ProBar.Value = I
                    If split(I).StartsWith(" ") = True Then split(I) = split(I).Remove(0, 1)
                    Dim IsIn As Boolean = False
                    For J As Integer = 0 To InternetSearchObjects.Items.Count - 1
                        If InternetSearchObjects.Items(J).ToString.Contains(split(I).ToString) Then IsIn = True

                    Next
                    If IsIn = False Then InternetSearchObjects.Items.Add(FilterOut(split(I).ToString))
                Next
            Else
                InternetSearchObjects.Items.Add(FilterOut(Add))
            End If
        End If
        SetProgressBar(InternetSearchObjects.Items.Count - 1, 0, 0)
        For I As Integer = 0 To InternetSearchObjects.Items.Count - 1
            ProBar.Value = I
            Dim split() As String = InternetSearchObjects.Items(I).ToString.Split(" ")
            For J As Integer = 0 To UBound(split)
                Dim IsIn As Boolean = False
                For K As Integer = 0 To PageSearchObject.Items.Count - 1
                    If PageSearchObject.Items(K).ToString.Contains(split(J)) = True Then IsIn = True
                Next
                If IsIn = False Then PageSearchObject.Items.Add(FilterOut(split(J).ToString))
            Next
        Next
        ProBar.Value = 0
        CheckSearchObjects()
    End Sub
    Private Sub RemoveToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles RemoveToolStripMenuItem.Click
        Do While (InternetSearchObjects.SelectedItems.Count > 0)
            InternetSearchObjects.Items.Remove(InternetSearchObjects.SelectedItem)
        Loop
    End Sub
    '   PaSeOb
    Private Sub InsertToolStripMenuItem1_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles InsertToolStripMenuItem1.Click
        Dim Add As String = InputBox("Object to Search on the Pages" & vbCrLf & "Seperate Items are Created when you use: ,", "Add Page Search Object")
        If Add.Contains(",") = True Then
            Dim Split() As String = Add.Split(",")
            For J As Integer = 0 To UBound(Split)
                Dim IsIn As Boolean = False
                For K As Integer = 0 To PageSearchObject.Items.Count - 1
                    If PageSearchObject.Items(K).ToString.Contains(Split(J)) = True Then IsIn = True
                Next
                If IsIn = False Then PageSearchObject.Items.Add(FilterOut(Split(J).ToString))
            Next
        Else
            Dim IsIn As Boolean = False
            For K As Integer = 0 To PageSearchObject.Items.Count - 1
                If PageSearchObject.Items(K).ToString.Contains(Add) = True Then IsIn = True
            Next
            If IsIn = False Then PageSearchObject.Items.Add(FilterOut(Add))
        End If
        CheckSearchObjects()
    End Sub
    Private Sub RemoveToolStripMenuItem1_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles RemoveToolStripMenuItem1.Click
        Do While (PageSearchObject.SelectedItems.Count > 0)
            PageSearchObject.Items.Remove(PageSearchObject.SelectedItem)
        Loop
    End Sub
    '   FoIn
    Private Sub RemoveToolStripMenuItem2_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles RemoveToolStripMenuItem2.Click
        Do While (FoundInformation.SelectedItems.Count > 0)
            FoundInformation.Items.Remove(InternetSearchObjects.SelectedItem)
        Loop
    End Sub
    Private Sub Item_click() Handles FoundInformation.DoubleClick
        For I As Integer = 0 To Results.Count - 1
            Dim Thing As String = FoundInformation.SelectedItem.ToString
            If Thing.Contains(Results(I)) Then
                Dim split() As String = Results(I).Split("@ ")
                Try
                    Process.Start(split(1))
                Catch
                End Try
                Exit Sub
            End If
        Next
    End Sub
    Private Sub SelectAllToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles SelectAllToolStripMenuItem.Click
        For I As Integer = 0 To FoundInformation.Items.Count - 1
            FoundInformation.SetSelected(I, True)
        Next
    End Sub
    '   Save
    Private Sub Save() Handles SaveToolStripMenuItem.Click
        SaveFileDialog1.Filter = "Text Files (*.txt)| *.txt| All Files (*.*)| *.*"
        SaveFileDialog1.FileName = ""

        If SaveFileDialog1.ShowDialog = Windows.Forms.DialogResult.OK Then
            Dim Contents As String
            Dim Reader As StreamWriter
            Contents = "Internet Search Objects:"
            For I As Integer = 0 To InternetSearchObjects.Items.Count - 1
                Contents += "," & InternetSearchObjects.Items(I).ToString
            Next
            Contents += "," & vbNewLine & ","
            Contents += "Page Search Objects:"
            For I As Integer = 0 To PageSearchObject.Items.Count - 1
                Contents += "," & PageSearchObject.Items(I).ToString
            Next
            Contents += "," & vbNewLine & ","
            Contents += "Results:"
            For I As Integer = 0 To Results.Count - 1
                Contents += "," & Results(I).ToString & vbNewLine
            Next
            Dim Content() As String = Contents.Split(",")
            Try
                Reader = New StreamWriter(SaveFileDialog1.FileName)
                For I As Integer = 0 To UBound(Content)
                    Reader.WriteLine(Content(I))
                Next
                Reader.Close()
            Catch
            End Try

        End If
    End Sub
    '   Settings
    Private Sub SettingsToolStripMenuItem_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles SettingsToolStripMenuItem.Click
        If Settings.ShowDialog = Windows.Forms.DialogResult.OK Then ProText.Text = ""
    End Sub
    '   Results
    Private Sub CountItem() Handles ResultsToolStripMenuItem.Click
        ItemCountToolStripMenuItem.Text = "Items: " & FoundInformation.Items.Count
    End Sub
    Private Sub RemoveToolStripMenuItem3_Click(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles RemoveToolStripMenuItem3.Click
        If FindRemove.ShowDialog = Windows.Forms.DialogResult.OK Then
            Dim Search As String = FindRemove.TextBox1.Text
            Dim Checked As Boolean = FindRemove.CheckBox1.Checked
            Dim Number As Integer = FindRemove.NumericUpDown1.Value
            Dim CheckedB As Boolean = FindRemove.CheckBox2.Checked
            Dim NumberB As Integer = FindRemove.NumericUpDown2.Value
            For I As Integer = 0 To FoundInformation.Items.Count - 1
                If I >= FoundInformation.Items.Count - 1 Then Exit For
                If FoundInformation.Items(I).ToString.ToLower.Contains(Search.ToLower) Then FoundInformation.Items.RemoveAt(I)
                If Number >= FoundInformation.Items(I).ToString.Length And Checked = True Then FoundInformation.Items.RemoveAt(I)
                If NumberB <= FoundInformation.Items(I).ToString.Length And CheckedB = True Then FoundInformation.Items.RemoveAt(I)
            Next
        End If
    End Sub
    '   Search Engine
    Function GetListOfSearchEngine() As List(Of String)
        Dim Path As String = Application.StartupPath() & "\Search Engine.ais"
        If File.Exists(Path) = False Then
            MsgBox("Search Engine File is missing")
            Application.Exit()
        Else
            Dim Reader As StreamReader
            Dim Content As String = ""
            Try
                Reader = New StreamReader(Path)
                Content = Reader.ReadToEnd
            Catch ex As Exception
            End Try
            Content = Content.Replace(" ", "")
            Dim Split() As String = Content.Split(vbNewLine)
            Dim Results As New List(Of String)
            If UBound(Split) <> 0 Then
                For I As Integer = 0 To UBound(Split)
                    Dim Line As String = Split(I).Replace(" ", "")
                    If Line.StartsWith("<") = False Then Line = Line.Remove(0, 1)
                    Line = Line.Replace(vbNewLine, "")
                    If Line.StartsWith("<") And Line.StartsWith("</") = False And Line.StartsWith("<Filter>") = False _
                    And Line.StartsWith("</Filter>") = False And Line <> "<>" Then
                        Results.Add(Line.Replace("<", "").Replace(">", ""))
                    End If

                Next
                Return Results
            Else
                MsgBox("Search Engine File is Corrupted")
                Application.Exit()
            End If
        End If


    End Function
    Function GetSearchEngineFilterlist(ByVal SearchEngine As String) As List(Of String)
        Dim Results As New List(Of String)
        Dim Path As String = Application.StartupPath() & "\Search Engine.ais"
        If File.Exists(Path) = False Then
            MsgBox("Search Engine File is missing")
            Application.Exit()
        Else
            Dim Reader As StreamReader
            Dim Content As String = ""
            Try
                Reader = New StreamReader(Path)
                Content = Reader.ReadToEnd
            Catch ex As Exception
            End Try
            Content = Content.Replace(" ", "")
            Dim Split() As String = Content.Split(vbNewLine)
            For I As Integer = 0 To UBound(Split)
                If Split(I).Contains("<" & SearchEngine & ">") = True Then
                    For J As Integer = I To UBound(Split)
                        If Split(J).Contains("</Filter>") Then
                            For K = I + 3 To J - 1
                                If Split(K).StartsWith("<") = False Then Results.Add(Split(K).Remove(0, 1))
                            Next
                            Return Results
                            Exit Function
                        End If
                    Next
                End If
            Next
        End If
    End Function
    Function GetSearchEngineURL(ByVal SearchEngine As String) As String
        Dim Path As String = Application.StartupPath() & "\Search Engine.ais"
        If File.Exists(Path) = False Then
            MsgBox("Search Engine File is missing")
            Application.Exit()
        Else
            Dim Reader As StreamReader
            Dim Content As String = ""
            Try
                Reader = New StreamReader(Path)
                Content = Reader.ReadToEnd
            Catch ex As Exception
            End Try
            Content = Content.Replace(" ", "")
            Dim Split() As String = Content.Split(vbNewLine)
            For I As Integer = 0 To UBound(Split)
                If Split(I).Contains("<" & SearchEngine & ">") = True Then
                    Dim Res As String = Split(I + 1).Remove(0, 5).Replace(" ", "").Replace(vbNewLine, "")
                    Return Res
                    Exit Function
                End If
            Next
        End If
    End Function

    'Functions
    Function ResetStuff()
        ProBar.Value = 0
        ProcesStop = False
        ProText.Text = ""
        StartToolStripMenuItem.Enabled = True
    End Function
    Function FilterUrls(ByVal Urls() As String) As String()
        Dim CheckedUrls As String = ""
        If SearchEngineFilterlist.Count >= 0 Then
            For I As Integer = 0 To UBound(Urls)
                For J As Integer = 0 To SearchEngineFilterlist.Count - 1
                    Dim Isin As Boolean = False
                    If Urls(I).Contains(SearchEngineFilterlist(J)) Then Isin = True
                    If Isin = False Then CheckedUrls += Urls(I) & ","
                Next
            Next
        End If
        Return CheckedUrls.Split(",")
    End Function
    Function GetUrls(ByVal Content As String) As String()
        Dim urls As String
        Dim Quote As String = Convert.ToChar(34)
        For X As Integer = 1 To Content.Length
            Dim Search As String = Mid(Content, X, 6)
            If Search = "href=" & Quote Then
                Dim Xb As Integer = 7
                Do While (Content(X + Xb) <> Quote)
                    Xb += 1
                Loop
                urls += Mid(Content, X, Xb + 1) & ","
                X += Xb
            End If
        Next
        If urls <> "" Then
            urls = urls.Replace("href=" & Quote, "")
            urls = urls.Replace(Quote, "")
            If urls.EndsWith(",") = True Then urls = urls.Remove(urls.Length - 1, 1)
            Return urls.Split(",")
        End If
    End Function
    Function GetPage(ByVal Url As String) As String
        Dim Browser As New Net.WebClient
        Dim Bytes() As Byte
        Dim Text As String = ""
        Try
            Bytes = Browser.DownloadData(Url)
            Text = System.Text.ASCIIEncoding.ASCII.GetString(Bytes)
            Browser.Dispose()
        Catch
        End Try
        Return Text
    End Function
    Function GetSearchEnginePage(ByVal SearchObject As String, ByVal PageNumber As Integer) As String
        Dim Browser As New Net.WebClient
        Dim Url As String = (SearchEngineURL.Replace("SEARCHOBJECT", (SearchObject.Replace(" ", "+"))).Replace("PAGE", PageNumber & "0")).Replace(" ", "")
        Dim Text As String = ""
        Dim Bytes() As Byte
        If Url.StartsWith("http://") = False Then Url = "http://" & Url
        Bytes = Browser.DownloadData(Url.ToString.ToLower)
        Text = System.Text.ASCIIEncoding.ASCII.GetString(Bytes)

        Browser.Dispose()
        Return Text
    End Function
    Function SetProgressBar(ByVal Max As Integer, ByVal Min As Integer, Optional ByVal Value As Integer = 0)
        If Max <= 0 Then Max = 1
        If Min <= 0 Then Min = 0
        ProBar.Maximum = Max
        ProBar.Minimum = Min
    End Function
    Function UpdateStuff(Optional ByVal ProgressBar As Integer = 0, Optional ByVal ProgressText As String = "", Optional ByVal FormText As String = "Advanced Internet Search")
        ProBar.Value = ProgressBar
        ProText.Text = ProgressText
        Console.Writeline(ProgressText)
        Me.Text = FormText
        ProBar.Update()
        ProText.Update()
        Me.Update()
        MenuStrip1.Update()

    End Function
    Function GetString(ByVal SearchObject As String, ByVal Contents As String) As List(Of String)
        Dim Results As New List(Of String)
        For Index As Integer = 1 To Contents.Length
            Dim Word As String = Mid(Contents, Index, SearchObject.Length)
            If Word = SearchObject Then
                Dim startpoint As Integer = Index
                Dim Chars As Char = Mid(Contents, startpoint, 1)
                Dim Quote As String = Convert.ToChar(34)
                Do Until (Chars = ":" Or Chars = "." Or Chars = "," Or Chars = Quote Or Chars = ">" _
                          Or Chars = "<" Or Chars = "(" Or Chars = ")" Or Chars = vbNewLine Or Chars = Environment.NewLine Or Chars = vbCrLf)
                    startpoint -= 1
                    If startpoint = 0 Then Exit Do
                    Chars = Mid(Contents, startpoint, 1)

                Loop
                Dim Endpoint As Integer = Index
                Chars = Mid(Contents, Endpoint, 1)
                Do Until (Chars = ":" Or Chars = "." Or Chars = "," Or Chars = Quote Or Chars = ">" _
                          Or Chars = "<" Or Chars = "(" Or Chars = ")")
                    Endpoint += 1
                    If Endpoint > Contents.Length Then Exit Do
                    Chars = Mid(Contents, Endpoint, 1)

                Loop
                Results.Add(Mid(Contents, startpoint + 1, (Endpoint - startpoint)))
            End If


        Next
        Return Results
    End Function
    Function DetectUrl(ByVal Tekst As String, ByVal Search As String) As Boolean
        Dim Bool As Boolean = False
        If (Tekst.Contains("/") Or Tekst.Contains("%") Or Tekst.Contains("?")) And Not (Tekst.Contains(" ")) _
        And Tekst.Length > (1.5 * Search.Length) Then Bool = True
        Return Bool
    End Function
    Function GetFilterList() As List(Of String)
        Dim ByteStream As FileStream
        Dim CurrentByte As Integer
        Dim lngLoop As Long = 0
        Dim ByteContent() As Byte
        Dim Contents As String
        Dim FinalList As New List(Of String)
        If File.Exists("FilterLibrary.ais") = False Then
            FinalList.Add("")
            Return FinalList
            Exit Function
        End If
        Try
            ByteStream = File.OpenRead("FilterLibrary.ais")
            ReDim ByteContent(ByteStream.Length)

            Do While Not CurrentByte = -1
                CurrentByte = ByteStream.ReadByte()
                If CurrentByte <> -1 Then ByteContent(lngLoop) = CByte(CurrentByte)
                lngLoop += 1
            Loop

            Contents = Encoding.ASCII.GetString(ByteContent)
            ByteStream.Close()

        Catch ex As IOException
        End Try
        Contents = Contents.ToLower
        Dim Split() As String = Contents.Split("#")

        For I As Integer = 0 To UBound(Split)
            FinalList.Add(Split(I))
        Next
        Return FinalList
    End Function
    Function FilterOut(ByVal Input As String) As String
        For I As Integer = 0 To FilterOutList.Count - 1
            Input = Input.Replace(FilterOutList(I), "")
        Next
        Return Input
    End Function
    Function CheckSearchObjects()
        For I As Integer = 0 To PageSearchObject.Items.Count - 1
            If PageSearchObject.Items(I).ToString.Length = 0 Then PageSearchObject.Items.RemoveAt(I)
            If I >= PageSearchObject.Items.Count Then Exit For
        Next
        For I As Integer = 0 To InternetSearchObjects.Items.Count - 1
            If InternetSearchObjects.Items(I).ToString.Length = 0 Then InternetSearchObjects.Items.RemoveAt(I)
            If I >= InternetSearchObjects.Items.Count Then Exit For
        Next
    End Function
  
End Class


THE WHOLE CREDIT GOES TO THE ORIGINAL PROGRAMMER --- Huntondoom

0 comments:

Post a Comment