Reading a Web page from fwh

[alvaro533]

Good evening,

I'm trying to read a web page from my program. The address is:

http://aviationweather.gov/adds/metars/index.php?submit=1&station_ids=LEAL&chk_metars=on&chk_tafs=on&hoursStr=3

I use this code, which works fine with pages that are not php:

Code: Select all  Expand view

    // -------------------------------------------------------------------------- //
    function leeADDS()
    local site
    local cad:=""
    local out
    local oWebClient
    local oTimer,oDlg
    local cPagename


    site := "aviationweather.gov"
    cPagename:= "/adds/metars/index.php?submit=1&station_ids=LEAL&chk_metars=on&chk_tafs=on&hoursStr=3"

        oWebClient := TWebClient():New()
        oWebClient:Connect( site )


    DEFINE TIMER otimer INTERVAL 1000 ACTION if(comprueba(cad) , oDlg:end() , nil) OF app():ownd
    ACTIVATE TIMER otimer

    DEFINE DIALOG oDlg FROM 3, 3 TO 28, 79 title "Espera "+cPageName

    //    oWebClient:bOnConnect = { || MsgInfo( "Connected!" ) }
    //    oWebClient:bOnRead    = { | cData | cad += cData     }
        oWebClient:bOnRead    = { | cData |  cad += cData    }
        oWebClient:oSocket:SendData( "GET " + cPageName + " HTTP/1.0" + CRLF +"Host: "+site+ CRLF +CRLF )


    activate dialog oDlg centered

    out:= fcreate("meteo.txt",0)
    fwrite(out,cad+CRLF)
    fclose(out)

    RELEASE TIMER otimer

    return nil

    // ---------------------------------------------------------------------------- //
    static function comprueba(cad)
    local bien:= .t.


    if !cad="HTTP/1.1 200 OK"
    bien:= .f.
    endif


    if !("</html>"$cad)
    bien:= .f.
    endif

    return bien
     
 

But if the page is php I get an error. I haven't found anything related in the forum. Any help would be apreciated. Thank you and regards,

Alvaro

[Daniel Garcia-Gil]

http://forums.fivetechsupport.com/viewtopic.php?p=125774#p125774

[Rick Lipkin]

alvaro

Are you just trying to 'scrape' the page and retrieve the text ? .. If so I have several snipits of code for you to consider using Internet Explorer automation.

Rick Lipkin

[alvaro533]

Hi Rick,
Yes, would like to have the code. Thank you very much for your offer. I will retreive the info and "cook" it. Will you post the code here or do you want my email address?

Regards

Alvaro

[alvaro533]

Hi Daniel,

Sometimes when the page is long, I obtain estrange characters in the page.

For instance I obtain

Code: Select all  Expand view

<
5b4
/FONT>
 

Instead of

Code: Select all  Expand view

</FONT>

If I reload the page then I may get it right, but the estrange character may be found in a different place. Can it be fixed? Thank you very much

Alvaro

[Rick Lipkin]

Alvaro

A bit more code than you asked for .. this should give you a text result you can send to a file or just view.

Rick Lipkin

Code: Select all  Expand view

//-- IETest.prg

#Include "FiveWin.ch"

Static cSAY,oSay


//---------------
Func Main()

LOCAL oBtn1,oBtn2,oDlg,oRad,nRad,oRad1,nRad1,oGroup,oGroup1
LOCAL cUrl,oUrl,oSay1,cSay1
LOCAL cFile,aDir,nStart,cDefa

//-- get timestamp on .exe //

cFILE := GetModuleFileName( GetInstance() )
aDIR  := DIRECTORY( cFILE )

// where .exe started from is default directory //

nSTART := RAT( "\", cFILE )
cDEFA  := SUBSTR(cFILE,1,nSTART-1)

aDIR := NIL

SET DEFA to ( cDEFA )


cSay  := Space(30)
nRad  := 2
nRad1 := 2
cUrl  := "http://aviationweather.gov/adds/metars/index.php?submit=1&station_ids="
cUrl  += "LEAL&chk_metars=on&chk_tafs=on&hoursStr=3"+space(80)
cSay1 := "Enter Web Address or Url to Scrape Page"

DEFINE DIALOG oDlg                             ;
       FROM 6, 2 to 30, 55                     ;
       TITLE "IE Automation Page Scrape Test"  ;
       STYLE nOr( WS_POPUP,WS_CAPTION,WS_THICKFRAME )

   @ .5,2    GROUP oGroup  to 2.8,12 of oDlg
   @ 1,3     RADIO oRad  var nRad  ITEMS "IE Visible ", "IE InVisible" SIZE 40,12

   @ 3.0,2   GROUP oGroup1 to 5.3,15 of oDlg
   @ 4,3     RADIO oRad1 var nRad1 ITEMS "Write Page Scrape to File ", "View Page Scrape " SIZE 70,12

   @ 5.3,2.5 Say oSay1 var cSay1 of oDlg

   @ 7,1.8   Get oUrl var cUrl of oDlg MULTILINE SIZE 170,25 Update

   @ 8.5,2.5 Say oSay var cSay of oDlg


   @ 8.5,16 BUTTON oBtn2 Prompt "&Ok"         ;
       SIZE 30,25 of oDLG                   ;
       ACTION ( _DoIt(nRad,nRad1,oDlg,oUrl,@cUrl) )

   @ 8.5,23 BUTTON oBtn1 Prompt "&Cancel"     ;
       SIZE 30,25 of oDLG                   ;
       ACTION ( oDlg:END() ) DEFAULT

ACTIVATE DIALOG oDlg


//--------------
Static Func _DoIt( nRad, nRad1, oDlg, oUrl, cUrl )

LOCAL Ie,nNum,cText,nLoop,Saying,cDefa,nHandle

cDEFA := SET(7)

If cUrl = " "
   Saying := "Sorry ... Web address can not be blank"
   MsgInfo( saying )
   cUrl := space(200)
   oUrl:ReFresh()
 * oUrl:SetFocus()
   SysReFresh()
   Return(.f.)
Endif

cUrl := alltrim( cUrl )

cSAY := "Creating Browser Object"
oSay:ReFresh()
SysReFresh()

Try
   IE := CreateObject("InternetExplorer.Application")
Catch
   cSAY := "Cound not Create Browser Object"
   oSay:ReFresh()
   SysReFresh()
   SysWait(1)
   oDlg:End()
   Return(.f.)
End Try

If nRad = 1
   IE:Visible := .t.
Else
   IE:Visible := .f.
Endif

cSAY := "Creating Browser Object  DONE"
oSay:ReFresh()
SysReFresh()

IE:Navigate( cURL )

nNUM := 0

Do While IE:Readystate <> 4
   SysWait(1)
   nNUM++
   if nNUM = 20 .and. IE:Readystate <> 4
      exit
   endif
   SysRefresh()
   Loop
Enddo

cSAY := "Opening Web Page DONE"
oSay:ReFresh()
SysReFresh()
SysWait(.5)

cSAY := "Gathering Page Information "
oSay:ReFresh()
SysReFresh()

try
  cTEXT := lower(IE:document:documentElement:outerTEXT)
catch
  cSAY := "Can not open Page Information "
  oSay:ReFresh()
  SysReFresh()

  try
    IE:Quit()
  catch
  end try

  IE := NIL
  SysReFresh()
  SysWait(1)
  oDlg:End()
  Return(.f.)

end try

SysWait( 3)

cSAY := "Gathering Page Information DONE"
oSay:ReFresh()
SysReFresh()


// take out all the control char
/*

cSAY := "Taking out all the Control Char"
oSay:ReFresh()
SysReFresh()

nLOOP := 0
DO WHILE .T.
   IF AT( (chr(13)+chr(10)), cTEXT )  > 0
      cTEXT := STRTRAN( cTEXT, (chr(13)+chr(10)), space(1) )
   ENDIF
   nLOOP++
   IF nLOOP > 500
      EXIT
   ENDIF
ENDDO

cSAY := "Taking out all the Control Char"
oSay:ReFresh()
SysReFresh()
SysWait(1)

*/

Try
   IE:Quit()
Catch
End Try

If nRad1 = 1

     cSAY := "Writing Page Scrape to File"
     oSay:ReFresh()
     SysReFresh()

     FERASE( cDefa+"\SCRAPE.TXT" )

     nHANDLE := FCREATE( cDefa+"\SCRAPE.TXT", 0 )
     FWRITE( nHANDLE, cText )   // write out the file
     FCLOSE( nHANDLE )

     cSAY := "Writing Page Scrape to File DONE"
     oSay:ReFresh()
     SysReFresh()

     WinExec( "NOTEPAD "+cDEFA+"\SCRAPE.TXT", 1 )

Else

   msginfo( ctext )

Endif

cUrl := Space(200)
oUrl:ReFresh()
SysReFresh()

Return(.t.)

//---- end



 

[alvaro533]

Thank you Rick. I'll test it now.
Alvaro