FIXING DispatchingTaskScheduler:1071

 
REQUEST

 
SYMPTOMS

Host=dbpilot.net
Target type=Agent 
Target name=dbpilot.net:3872 
Categories=Diagnostics 
Message=Internal error detected: java.lang.Throwable: oracle.sysman.gcagent.task.scheduler.DispatchingTaskScheduler:1071. 
Severity=Critical 
Event reported time=Feb 12, 2018 7:43:23 AM MSK 
Customer Support Identifier=10000001
Operating System=Linux
Platform=x86_64
Associated Incident Id=47959 
Associated Incident Status=New 
Associated Incident Owner= 
Associated Incident Acknowledged By Owner=No 
Associated Incident Priority=None 
Associated Incident Escalation Level=0 
Event Type=Metric Alert 
Event name=adrIncidentError:diagnosticIncident 
Metric Group=Incident
Metric=Diagnostic Incident
Metric value=diagnosticIncident
Key Value=518
Key Column 1=Incident ID
Rule Name=Notifications All,rule 156 
Rule Owner=SYSMAN 
Update Details:
Internal error detected: java.lang.Throwable:oracle.sysman.gcagent.task.scheduler.DispatchingTaskScheduler:1071.

 
GATHER DETAILS

-- Total number of generated erorrs

SET LINES 300
SET PAGES 999
COL HOST FOR A40
COL TYPE FOR A10 
SELECT TARGET_NAME HOST, TARGET_TYPE TYPE, COUNT(TARGET_NAME) "NUMBER OF SENT ERRORS" FROM 
MGMT_VIEW.MGMT$ALERT_NOTIF_LOG WHERE MESSAGE LIKE '%DispatchingTaskScheduler:1071%'
GROUP BY TARGET_NAME, TARGET_TYPE ORDER BY COUNT(TARGET_NAME);

HOST               TYPE       NUMBER OF SENT ERRORS
------------------ ---------- ---------------------
dbpilot.net:3872   oracle_emd                   125

-- Generated errors for the last month by days

SET LINES 300
SET PAGES 999
COL HOST FOR A40
COL TYPE FOR A10 
SELECT TO_DATE(COLLECTION_TIMESTAMP,'DD-MON-YYYY') "DATE", TARGET_NAME HOST, COUNT(TARGET_NAME) "NUMBER OF SENT ERRORS" FROM 
MGMT_VIEW.MGMT$ALERT_NOTIF_LOG WHERE MESSAGE LIKE '%DispatchingTaskScheduler:1071%'
GROUP BY TO_DATE(COLLECTION_TIMESTAMP,'DD-MON-YYYY'), TARGET_NAME ORDER BY 1;

DATE               HOST                NUMBER OF SENT ERRORS
------------------ ------------------- ---------------------
08-FEB-18          dbpilot.net:3872                       25
09-FEB-18          dbpilot.net:3872                       25
10-FEB-18          dbpilot.net:3872                       25
11-FEB-18          dbpilot.net:3872                       25
12-FEB-18          dbpilot.net:3872                       25

 
SOLUTION

Follow the steps provided below:

1. Apply the agent environment, eg

[oracle@dbpilot ~] cat agent.env

###############################
## Victor Khalturin
## Agent environment file 
###############################

unset ORACLE_HOME
unset AGENT_HOME
unset AGENT_INST
unset PATH
unset PATCH_HOME

export ORACLE_HOME=/agent13c/agent_13.1.0.0.0
export AGENT_HOME=/agent13c/agent_13.1.0.0.0
export AGENT_INST=/agent13c/agent_inst
export PATH=$AGENT_HOME/bin:$AGENT_HOME/OPatch:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/oracle/.local/bin:/home/oracle/bin
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
export PS1="\[\e[32;1m\]\u@\h\[\e[m\]|\A|\[\e[0;33m\]\w:\[\e[m\]\[\e[31;1m\]\[\e[m\]"
export PATCH_HOME=/home/oracle/.env/patches

echo
echo "    ======================================================"
echo "    ORACLE_HOME  : ${ORACLE_HOME}"
echo "    AGENT_HOME   : ${AGENT_HOME}"
echo "    AGENT_INST   : ${AGENT_INST}"
echo "    PATCH_HOME   : ${PATCH_HOME}"
echo "    NLS_LANG     : ${NLS_LANG}"
echo "    ======================================================"
echo

[oracle@dbpilot ~] source agent.env


    ======================================================
    ORACLE_HOME  : /agent13c/agent_13.1.0.0.0
    AGENT_HOME   : /agent13c/agent_13.1.0.0.0
    AGENT_INST   : /agent13c/agent_inst
    PATCH_HOME   : /home/oracle/.env/patches
    NLS_LANG     : AMERICAN_AMERICA.AL32UTF8
    ======================================================

2. Backup $AGENT_INST/sysman/config/emd.properties file

[oracle@dbpilot ~] cp $AGENT_INST/sysman/config/emd.properties $AGENT_INST/sysman/config/emd.properties_$(date +%d%m%y)
[oracle@dbpilot ~] ls -ltr --time-style='+' $AGENT_INST/sysman/config/emd.properties*
-rw-r-----. 1 oracle dba 8734  /agent13c/agent_inst/sysman/config/emd.properties
-rw-r-----. 1 oracle dba 8734  /agent13c/agent_inst/sysman/config/emd.properties_120218

3. Add the following property in emd.properties file: disableDuplicateTaskCheck= true

[oracle@dbpilot ~] egrep disableDuplicateTaskCheck $AGENT_INST/sysman/config/emd.properties
disableDuplicateTaskCheck=true

4. Stop the agent

[oracle@dbpilot ~ emctl stop agent
Oracle Enterprise Manager Cloud Control 13c Release 1
Copyright (c) 1996, 2015 Oracle Corporation.  All rights reserved.
Stopping agent ... stopped.

5. Apply the patch 23208577 or higher to the agent

[oracle@dbpilot ~] ls
p23208577_131000_Generic.zip
[oracle@dbpilot ~] unzip -o p23208577_131000_Generic.zip && cd 23208577 && opatch apply && cd && opatch lsinventory |grep applied

OPatch succeeded.
Patch  23208577     : applied on Mon Feb 12 16:11:47 MSK 2018
Patch  18421945     : applied on Mon Jan 15 15:21:47 MSK 2018

6. Start the agent

[oracle@dbpilot ~] emctl start agent
Oracle Enterprise Manager Cloud Control 13c Release 1
Copyright (c) 1996, 2015 Oracle Corporation.  All rights reserved.
Starting agent .................... started.

7. Query OMS database to confirm the patch is applied. It will take some time the table is updated with the patch information

SET LINES 200 
SET PAGES 999COL PSU FOR A3
COL PATCH FOR A10
COL BUGS FOR A40
COL PATCH_RELEASE FOR A15
COL PLATFORM FOR A15
COL HOST FOR A10
COL HOME_NAME FOR A15
COL HOME_LOCATION FOR A60

SELECT IS_PSU PSU, PATCH, PATCH_RELEASE, PLATFORM, HOST, HOME_NAME, HOME_LOCATION, INSTALLATION_TIME
FROM MGMT_VIEW.MGMT$APPLIED_PATCHES WHERE PATCH='23208577' ORDER BY INSTALLATION_TIME;

P PATCH     PATCH_RELEASE PLATFORM      HOST        HOME_NAME  HOME_LOCATION              INSTALLATION_TIME
- --------- ------------- ------------- ----------- ---------- -------------------------- ------------------
N 23208577  13.1.0.0.0    Linux x86-64  dbpilot.net agent13c1  /agent13c/agent_13.1.0.0.0 12-FEB-2018

 
REFERENCES

EM 13c : Agent Generates Many Incidents with “Internal error detected: java.lang.Throwable:oracle.sysman.gcagent.task.scheduler.DispatchingTaskSchedule” (Doc ID 2108408.1)