Date: Wed, 8 Nov 2000 10:13:52 -0800
Reply-To: Jean-Pierre Le Cruguel <lecruguel@epid.jgh.mcgill.ca>
Sender: "SAS(r) Discussion" <SAS-L@LISTSERV.UGA.EDU>
From: Jean-Pierre Le Cruguel <lecruguel@EPID.JGH.MCGILL.CA>
Subject: Re: Save LOG Clear LOG Save LOG ...
Content-Type: text/plain; charset="iso-8859-1"
----- Original Message -----
From: <Jonathan_Goldberg@mastercard.com>
To: <SAS-L@LISTSERV.UGA.EDU>; <lecruguel@EPID.JGH.MCGILL.CA>
Sent: Tuesday, November 07, 2000 2:04 PM
Subject: Re: Save LOG Clear LOG Save LOG ...
>
>
> JP <R_E_M_O_V_E_lecruguel@EPID.JGH.MCGILL.CA> asks:
>
> >I am doing some simulations, my simulations are highly iterative: I do
the
> >same things (MEANS, LOGISTIC and few data steps) many many time (1
million
> >time: I know it is a lot, I do some repetition of an iterative process
with
> >some bootstrap). It rans quite ok, but at a random time, I often get:
> >
> >error messages deleted
> >
> >Then it stops. Often I am at >50% of my simulation.
> >(splitting it up did not help yet)
> >The SAS TS ak me to keep the log, which I can't (too big).
> >Is there a way to keep it, the best thing would be to rewrite it from
> >scratch periodically.
> >As I use
> > OPTIONS ERRORABEND;
> > OPTIONS ERRORS = 1;
> >I would only have the log before the crash.
> >
> >Any example. I suppose Proc Printto would be the thing.
> >Thanks,
> .
> >JP
>
> If I understand the question, the problem is to keep logs from runs that
fail
> without saving all of them.
>
Exactly,
> It makes a big difference how your job is organized. If you have a program
that
> is repeatedly executed you should run a tiny program between executions of
the
> simulation. This program would check the return code from the previous
> simluation run. If it ended abnormally, your program would rename or move
the
> log. Otherwise, allow it to be overwritten.
>
My process is highly iterative. So having a proc printto well placed in the
do loops should do the trick, as long as there is an option to rewrite over
it.
I use OPTIONS ERRORABEND and ERROR = 1;
so with a renewing proc printto well placed in a do loop, we should be able
to keep the log at the crash time.
> If you are generating one gigantic log, from a single execution, then you
need
> to add macro code to test the exit status of failure-prone steps. This
would
> allow you to end early and save the part of the log with the error. This
would
> create a more flexible, hand-coded version of ERRORABEND. It is possible
to
> created automatic re-start arrangements; exactly how depends on your
platform.
> I'd rather face this problem under UNIX than anywhere else with which I'm
> familar.
>
> This is all a bit short on specifics; for those we need more details
about your
> system and code.
>
I am in UNIX, SAS v8.
here is the calling pgm:
LIBNAME mcat1 "/home/cruguel/sim3/macro2";
OPTIONS MSTORED SASMSTORE=mcat1;
OPTIONS MPRINT;
OPTIONS SOURCE SOURCE2;
OPTIONS NOMPRINT;
OPTIONS ERRORABEND;
OPTIONS ERRORS = 1;
OPTIONS NONOTES;
OPTIONS NOSOURCE NOSOURCE2;
LIBNAME ssd "/home/cruguel/sim3";
%lancegen(30,ssd.n3b1501a4x3,100,ssd.n3b1501b4x3,ssd.n3b1501c4x3,200 ,1.5
,1 ,0.1, w,iv,2 ,4 );
and here is the macro code called in the pgm above:
(for those interested :description: repeated proc logistic with a decreasing
range of obs. Then I do a proc reg on beta1(i) (from repeated logistics)
explained by 1/varX (i) i being a subsamble (decrasing number of
obs)
I repeat that 30 times. I also do some 100 bootstrap within each
repetitions. )
The macro PE is the iterative one in the heart of the loops.
LIBNAME mcat1 "/home/cruguel/sim3/macro2";
OPTIONS MSTORED SASMSTORE=mcat1;
%MACRO lancegen(R,OUTPE,B,bootpes,OUTBOOT,
nbobs,beta1,v_x,v_u,varobs,iv,forceiv,pas) /STORE DES =
'macro lancemet Gal PE repetees + bootstrap';
DATA &outpe;
SET _NULL_;
RUN;
DATA &bootpes;
SET _NULL_;
RUN;
DATA &outboot;
SET _NULL_;
RUN;
%DO j = 1 %TO &R;
%genered(dbase);
%pe(dbase,pe);
DATA &outpe;
SET &outpe pe;
RUN;
DATA peboot;
SET _NULL_;
RUN;
%DO k = 1 %TO &B;
%bootdat(bootout);
DATA peboot;
SET peboot bootout;
RUN;
DATA &bootpes;
SET &bootpes bootout (IN = z1);
IF z1 THEN iterbt = &k;
RUN;
%END;
PROC MEANS DATA = peboot (RENAME = (_TYPE_ = est)) NOPRINT;
VAR intercept;
CLASS est limite;
OUTPUT OUT = peboot2 (WHERE = ( est IN('PARMS','SEB') AND limite
NE .)) MEAN = b1hat VAR = varbtb1 STD = sebbtb1 N = B;
RUN;
DATA &outboot;
SET &outboot peboot2 ;
RUN;
%END;
%MEND lancegen;
%MACRO genered(dataout) /STORE DES = 'genere data';
DATA &dataout (KEEP = id ybin w &iv);
DO id=1 TO &nbobs;
x = RANNOR(INT(TIME())+1)*&v_x**0.5;
w = x + RANNOR(INT(TIME())+3)*&v_u**0.5;
y= 0.25 + &beta1*x;
py=1/(1+EXP(-y));
ybin=RANBIN(INT(TIME())+4,1,py);
&iv= &forceiv*x + RANNOR(INT(TIME())+1)*&forceiv;
OUTPUT;
END;
RUN;
%MEND genered;
%MACRO pe(din,dout) /STORE DES = 'fait la PE jusqu au reg lin';
PROC SORT DATA =&din OUT = courant;
BY &iv; * par la variable instrumentale;
RUN;
DATA temp; *initialisation du jeu de donnée de sortie;
SET _NULL_;
RUN;
%DO i = 0 %TO
%sysfunc(min(%sysfunc(int(%sysevalf((0.50*&nbobs)/(2*&pas)))),150));
%LET debut = %EVAL(1+(&i*&pas));
%LET fin = %EVAL(&nbobs-(&i*&pas));
DATA courant;
SET courant;
IF NOT(&debut LE _N_ LE &fin) THEN exclus = 1;
RUN;
PROC MEANS DATA = courant NOPRINT;
VAR &varobs;
OUTPUT OUT = output1b (WHERE = (_TYPE_=0) KEEP = vartot _TYPE_
_FREQ_) VAR = vartot;
WHERE exclus NE 1;
RUN;
PROC LOGISTIC DATA = courant DESCENDING NOPRINT OUTEST = output2
(KEEP = &varobs);
MODEL ybin = &varobs;
WHERE exclus NE 1;
RUN;
DATA output;
SET output1b (DROP= _TYPE_); SET output2;
RUN;
DATA temp;
SET temp output;
iter = _N_;
RUN;
DM 'Clear log';
%END;
DATA reg;
SET temp;
invart = 1/vartot;
RUN;
PROC REG DATA = reg NOPRINT OUTEST=sortie1 (DROP = _MODEL_ _RMSE_ _DEPVAR_
w);
MODEL w = invart/OUTSEB;
RUN;
PROC REG DATA = reg NOPRINT OUTEST=sortie2 (DROP = _MODEL_ _RMSE_ _DEPVAR_
w);
MODEL w = invart /OUTSEB;
WHERE _FREQ_ GE %sysfunc(int(%sysevalf(&nbobs*0.55)));
RUN;
PROC REG DATA = reg NOPRINT OUTEST=sortie3 (DROP = _MODEL_ _RMSE_ _DEPVAR_
w);
MODEL w = invart /OUTSEB;
WHERE _FREQ_ GE %sysfunc(int(%sysevalf(&nbobs*0.60)));
RUN;
PROC REG DATA = reg NOPRINT OUTEST=sortie4 (DROP = _MODEL_ _RMSE_ _DEPVAR_
w);
MODEL w = invart /OUTSEB;
WHERE _FREQ_ GE %sysfunc(int(%sysevalf(&nbobs*0.65)));
RUN;
PROC REG DATA = reg NOPRINT OUTEST=sortie5 (DROP = _MODEL_ _RMSE_ _DEPVAR_
w);
MODEL w = invart /OUTSEB;
WHERE _FREQ_ GE %sysfunc(int(%sysevalf(&nbobs*0.70)));
RUN;
PROC REG DATA = reg NOPRINT OUTEST=sortie6 (DROP = _MODEL_ _RMSE_ _DEPVAR_
w);
MODEL w = invart /OUTSEB;
WHERE _FREQ_ GE %sysfunc(int(%sysevalf(&nbobs*0.75)));
RUN;
PROC REG DATA = reg NOPRINT OUTEST=sortie7 (DROP = _MODEL_ _RMSE_ _DEPVAR_
w);
MODEL w = invart /OUTSEB;
WHERE _FREQ_ GE %sysfunc(int(%sysevalf(&nbobs*0.80)));
RUN;
PROC REG DATA = reg NOPRINT OUTEST=sortie8 (DROP = _MODEL_ _RMSE_ _DEPVAR_
w);
MODEL w = invart /OUTSEB;
WHERE _FREQ_ GE %sysfunc(int(%sysevalf(&nbobs*0.85)));
RUN;
PROC REG DATA = reg NOPRINT OUTEST=sortie9 (DROP = _MODEL_ _RMSE_ _DEPVAR_
w);
MODEL w = invart /OUTSEB;
WHERE _FREQ_ GE %sysfunc(int(%sysevalf(&nbobs*0.90)));
RUN;
DATA &dout;
SET sortie1 (IN=z1) sortie2 (IN=z2) sortie3 (IN=z3) sortie4 (IN=z4)
sortie5 (IN=z5) sortie6 (IN=z6) sortie7 (IN=z7) sortie8 (IN=z8) sortie9
(IN=z9) ;
IF z1 THEN limite = %sysfunc(int(%sysevalf(&nbobs*0.50)));
IF z2 THEN limite = %sysfunc(int(%sysevalf(&nbobs*0.55)));
IF z3 THEN limite = %sysfunc(int(%sysevalf(&nbobs*0.60)));
IF z4 THEN limite = %sysfunc(int(%sysevalf(&nbobs*0.65)));
IF z5 THEN limite = %sysfunc(int(%sysevalf(&nbobs*0.70)));
IF z6 THEN limite = %sysfunc(int(%sysevalf(&nbobs*0.75)));
IF z7 THEN limite = %sysfunc(int(%sysevalf(&nbobs*0.80)));
IF z8 THEN limite = %sysfunc(int(%sysevalf(&nbobs*0.85)));
IF z9 THEN limite = %sysfunc(int(%sysevalf(&nbobs*0.90)));
RUN;
%MEND pe;
%MACRO bootdat(boutout) /STORE DES = 'genere data bootstrap';
DATA boot1 (KEEP = id);
DO i = 1 TO &nbobs;
id = CEIL(RANUNI(INT(TIME()))*&nbobs);
OUTPUT;
END;
RUN;
PROC SORT DATA = boot1 OUT = boot2;
BY id;
RUN;
DATA boot3;
MERGE boot2 (IN=z1) dbase;
BY id;
IF z1;
RUN;
%pe(boot3,bootout);
%MEND bootdat;
