001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.hadoop.mapred;
020    
021    import java.io.DataInput;
022    import java.io.DataOutput;
023    import java.io.IOException;
024    import java.util.ArrayList;
025    import java.util.Collection;
026    import java.util.Collections;
027    
028    import org.apache.hadoop.classification.InterfaceAudience;
029    import org.apache.hadoop.classification.InterfaceStability;
030    import org.apache.hadoop.io.Text;
031    import org.apache.hadoop.io.Writable;
032    import org.apache.hadoop.io.WritableUtils;
033    import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
034    import org.apache.hadoop.util.StringInterner;
035    
036    /**
037     * Status information on the current state of the Map-Reduce cluster.
038     * 
039     * <p><code>ClusterStatus</code> provides clients with information such as:
040     * <ol>
041     *   <li>
042     *   Size of the cluster. 
043     *   </li>
044     *   <li>
045     *   Name of the trackers. 
046     *   </li>
047     *   <li>
048     *   Task capacity of the cluster. 
049     *   </li>
050     *   <li>
051     *   The number of currently running map & reduce tasks.
052     *   </li>
053     *   <li>
054     *   State of the <code>JobTracker</code>.
055     *   </li>
056     *   <li>
057     *   Details regarding black listed trackers.
058     *   </li>
059     * </ol></p>
060     * 
061     * <p>Clients can query for the latest <code>ClusterStatus</code>, via 
062     * {@link JobClient#getClusterStatus()}.</p>
063     * 
064     * @see JobClient
065     */
066    @InterfaceAudience.Public
067    @InterfaceStability.Stable
068    public class ClusterStatus implements Writable {
069      /**
070       * Class which encapsulates information about a blacklisted tasktracker.
071       *  
072       * The information includes the tasktracker's name and reasons for
073       * getting blacklisted. The toString method of the class will print
074       * the information in a whitespace separated fashion to enable parsing.
075       */
076      public static class BlackListInfo implements Writable {
077    
078        private String trackerName;
079    
080        private String reasonForBlackListing;
081        
082        private String blackListReport;
083        
084        BlackListInfo() {
085        }
086        
087    
088        /**
089         * Gets the blacklisted tasktracker's name.
090         * 
091         * @return tracker's name.
092         */
093        public String getTrackerName() {
094          return trackerName;
095        }
096    
097        /**
098         * Gets the reason for which the tasktracker was blacklisted.
099         * 
100         * @return reason which tracker was blacklisted
101         */
102        public String getReasonForBlackListing() {
103          return reasonForBlackListing;
104        }
105    
106        /**
107         * Sets the blacklisted tasktracker's name.
108         * 
109         * @param trackerName of the tracker.
110         */
111        void setTrackerName(String trackerName) {
112          this.trackerName = trackerName;
113        }
114    
115        /**
116         * Sets the reason for which the tasktracker was blacklisted.
117         * 
118         * @param reasonForBlackListing
119         */
120        void setReasonForBlackListing(String reasonForBlackListing) {
121          this.reasonForBlackListing = reasonForBlackListing;
122        }
123    
124        /**
125         * Gets a descriptive report about why the tasktracker was blacklisted.
126         * 
127         * @return report describing why the tasktracker was blacklisted.
128         */
129        public String getBlackListReport() {
130          return blackListReport;
131        }
132    
133        /**
134         * Sets a descriptive report about why the tasktracker was blacklisted.
135         * @param blackListReport report describing why the tasktracker 
136         *                        was blacklisted.
137         */
138        void setBlackListReport(String blackListReport) {
139          this.blackListReport = blackListReport;
140        }
141    
142        @Override
143        public void readFields(DataInput in) throws IOException {
144          trackerName = StringInterner.weakIntern(Text.readString(in));
145          reasonForBlackListing = StringInterner.weakIntern(Text.readString(in));
146          blackListReport = StringInterner.weakIntern(Text.readString(in));
147        }
148    
149        @Override
150        public void write(DataOutput out) throws IOException {
151          Text.writeString(out, trackerName);
152          Text.writeString(out, reasonForBlackListing);
153          Text.writeString(out, blackListReport);
154        }
155    
156        @Override
157        /**
158         * Print information related to the blacklisted tasktracker in a
159         * whitespace separated fashion.
160         * 
161         * The method changes any newlines in the report describing why
162         * the tasktracker was blacklisted to a ':' for enabling better
163         * parsing.
164         */
165        public String toString() {
166          StringBuilder sb = new StringBuilder();
167          sb.append(trackerName);
168          sb.append("\t");
169          sb.append(reasonForBlackListing);
170          sb.append("\t");
171          sb.append(blackListReport.replace("\n", ":"));
172          return sb.toString();
173        }
174        
175      }
176      
177      public static final long UNINITIALIZED_MEMORY_VALUE = -1;
178      
179      private int numActiveTrackers;
180      private Collection<String> activeTrackers = new ArrayList<String>();
181      private int numBlacklistedTrackers;
182      private int numExcludedNodes;
183      private long ttExpiryInterval;
184      private int map_tasks;
185      private int reduce_tasks;
186      private int max_map_tasks;
187      private int max_reduce_tasks;
188      private JobTrackerStatus status;
189      private Collection<BlackListInfo> blacklistedTrackersInfo =
190        new ArrayList<BlackListInfo>();
191      private int grayListedTrackers;
192    
193      ClusterStatus() {}
194      
195      /**
196       * Construct a new cluster status.
197       * 
198       * @param trackers no. of tasktrackers in the cluster
199       * @param blacklists no of blacklisted task trackers in the cluster
200       * @param ttExpiryInterval the tasktracker expiry interval
201       * @param maps no. of currently running map-tasks in the cluster
202       * @param reduces no. of currently running reduce-tasks in the cluster
203       * @param maxMaps the maximum no. of map tasks in the cluster
204       * @param maxReduces the maximum no. of reduce tasks in the cluster
205       * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
206       */
207      ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, 
208                    int maps, int reduces,
209                    int maxMaps, int maxReduces, JobTrackerStatus status) {
210        this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps, 
211             maxReduces, status, 0);
212      }
213    
214      /**
215       * Construct a new cluster status.
216       * 
217       * @param trackers no. of tasktrackers in the cluster
218       * @param blacklists no of blacklisted task trackers in the cluster
219       * @param ttExpiryInterval the tasktracker expiry interval
220       * @param maps no. of currently running map-tasks in the cluster
221       * @param reduces no. of currently running reduce-tasks in the cluster
222       * @param maxMaps the maximum no. of map tasks in the cluster
223       * @param maxReduces the maximum no. of reduce tasks in the cluster
224       * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
225       * @param numDecommissionedNodes number of decommission trackers
226       */
227      ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, int maps,
228          int reduces, int maxMaps, int maxReduces, JobTrackerStatus status,
229          int numDecommissionedNodes) {
230        this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps,
231          maxReduces, status, numDecommissionedNodes, 0);
232      }
233    
234      /**
235       * Construct a new cluster status.
236       * 
237       * @param trackers no. of tasktrackers in the cluster
238       * @param blacklists no of blacklisted task trackers in the cluster
239       * @param ttExpiryInterval the tasktracker expiry interval
240       * @param maps no. of currently running map-tasks in the cluster
241       * @param reduces no. of currently running reduce-tasks in the cluster
242       * @param maxMaps the maximum no. of map tasks in the cluster
243       * @param maxReduces the maximum no. of reduce tasks in the cluster
244       * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
245       * @param numDecommissionedNodes number of decommission trackers
246       * @param numGrayListedTrackers number of graylisted trackers
247       */
248      ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, int maps,
249          int reduces, int maxMaps, int maxReduces, JobTrackerStatus status,
250          int numDecommissionedNodes, int numGrayListedTrackers) {
251        numActiveTrackers = trackers;
252        numBlacklistedTrackers = blacklists;
253        this.numExcludedNodes = numDecommissionedNodes;
254        this.ttExpiryInterval = ttExpiryInterval;
255        map_tasks = maps;
256        reduce_tasks = reduces;
257        max_map_tasks = maxMaps;
258        max_reduce_tasks = maxReduces;
259        this.status = status;
260        this.grayListedTrackers = numGrayListedTrackers;
261      }
262    
263      /**
264       * Construct a new cluster status.
265       * 
266       * @param activeTrackers active tasktrackers in the cluster
267       * @param blacklistedTrackers blacklisted tasktrackers in the cluster
268       * @param ttExpiryInterval the tasktracker expiry interval
269       * @param maps no. of currently running map-tasks in the cluster
270       * @param reduces no. of currently running reduce-tasks in the cluster
271       * @param maxMaps the maximum no. of map tasks in the cluster
272       * @param maxReduces the maximum no. of reduce tasks in the cluster
273       * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
274       */
275      ClusterStatus(Collection<String> activeTrackers, 
276          Collection<BlackListInfo> blacklistedTrackers,
277          long ttExpiryInterval,
278          int maps, int reduces, int maxMaps, int maxReduces, 
279          JobTrackerStatus status) {
280        this(activeTrackers, blacklistedTrackers, ttExpiryInterval, maps, reduces, 
281             maxMaps, maxReduces, status, 0);
282      }
283    
284    
285      /**
286       * Construct a new cluster status.
287       * 
288       * @param activeTrackers active tasktrackers in the cluster
289       * @param blackListedTrackerInfo blacklisted tasktrackers information 
290       * in the cluster
291       * @param ttExpiryInterval the tasktracker expiry interval
292       * @param maps no. of currently running map-tasks in the cluster
293       * @param reduces no. of currently running reduce-tasks in the cluster
294       * @param maxMaps the maximum no. of map tasks in the cluster
295       * @param maxReduces the maximum no. of reduce tasks in the cluster
296       * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
297       * @param numDecommissionNodes number of decommission trackers
298       */
299      
300      ClusterStatus(Collection<String> activeTrackers,
301          Collection<BlackListInfo> blackListedTrackerInfo, long ttExpiryInterval,
302          int maps, int reduces, int maxMaps, int maxReduces,
303          JobTrackerStatus status, int numDecommissionNodes) {
304        this(activeTrackers.size(), blackListedTrackerInfo.size(),
305            ttExpiryInterval, maps, reduces, maxMaps, maxReduces, status,
306            numDecommissionNodes);
307        this.activeTrackers = activeTrackers;
308        this.blacklistedTrackersInfo = blackListedTrackerInfo;
309      }
310    
311      /**
312       * Get the number of task trackers in the cluster.
313       * 
314       * @return the number of task trackers in the cluster.
315       */
316      public int getTaskTrackers() {
317        return numActiveTrackers;
318      }
319      
320      /**
321       * Get the names of task trackers in the cluster.
322       * 
323       * @return the active task trackers in the cluster.
324       */
325      public Collection<String> getActiveTrackerNames() {
326        return activeTrackers;
327      }
328    
329      /**
330       * Get the names of task trackers in the cluster.
331       * 
332       * @return the blacklisted task trackers in the cluster.
333       */
334      public Collection<String> getBlacklistedTrackerNames() {
335        ArrayList<String> blacklistedTrackers = new ArrayList<String>();
336        for(BlackListInfo bi : blacklistedTrackersInfo) {
337          blacklistedTrackers.add(bi.getTrackerName());
338        }
339        return blacklistedTrackers;
340      }
341    
342      /**
343       * Get the names of graylisted task trackers in the cluster.
344       *
345       * The gray list of trackers is no longer available on M/R 2.x. The function
346       * is kept to be compatible with M/R 1.x applications.
347       *
348       * @return an empty graylisted task trackers in the cluster.
349       */
350      @Deprecated
351      public Collection<String> getGraylistedTrackerNames() {
352        return Collections.emptySet();
353      }
354    
355      /**
356       * Get the number of graylisted task trackers in the cluster.
357       *
358       * The gray list of trackers is no longer available on M/R 2.x. The function
359       * is kept to be compatible with M/R 1.x applications.
360       *
361       * @return 0 graylisted task trackers in the cluster.
362       */
363      @Deprecated
364      public int getGraylistedTrackers() {
365        return grayListedTrackers;
366      }
367    
368      /**
369       * Get the number of blacklisted task trackers in the cluster.
370       * 
371       * @return the number of blacklisted task trackers in the cluster.
372       */
373      public int getBlacklistedTrackers() {
374        return numBlacklistedTrackers;
375      }
376      
377      /**
378       * Get the number of excluded hosts in the cluster.
379       * @return the number of excluded hosts in the cluster.
380       */
381      public int getNumExcludedNodes() {
382        return numExcludedNodes;
383      }
384      
385      /**
386       * Get the tasktracker expiry interval for the cluster
387       * @return the expiry interval in msec
388       */
389      public long getTTExpiryInterval() {
390        return ttExpiryInterval;
391      }
392      
393      /**
394       * Get the number of currently running map tasks in the cluster.
395       * 
396       * @return the number of currently running map tasks in the cluster.
397       */
398      public int getMapTasks() {
399        return map_tasks;
400      }
401      
402      /**
403       * Get the number of currently running reduce tasks in the cluster.
404       * 
405       * @return the number of currently running reduce tasks in the cluster.
406       */
407      public int getReduceTasks() {
408        return reduce_tasks;
409      }
410      
411      /**
412       * Get the maximum capacity for running map tasks in the cluster.
413       * 
414       * @return the maximum capacity for running map tasks in the cluster.
415       */
416      public int getMaxMapTasks() {
417        return max_map_tasks;
418      }
419    
420      /**
421       * Get the maximum capacity for running reduce tasks in the cluster.
422       * 
423       * @return the maximum capacity for running reduce tasks in the cluster.
424       */
425      public int getMaxReduceTasks() {
426        return max_reduce_tasks;
427      }
428      
429      /**
430       * Get the JobTracker's status.
431       * 
432       * @return {@link JobTrackerStatus} of the JobTracker
433       */
434      public JobTrackerStatus getJobTrackerStatus() {
435        return status;
436      }
437      
438      /**
439       * Returns UNINITIALIZED_MEMORY_VALUE (-1)
440       */
441      @Deprecated
442      public long getMaxMemory() {
443        return UNINITIALIZED_MEMORY_VALUE;
444      }
445      
446      /**
447       * Returns UNINITIALIZED_MEMORY_VALUE (-1)
448       */
449      @Deprecated
450      public long getUsedMemory() {
451        return UNINITIALIZED_MEMORY_VALUE;
452      }
453    
454      /**
455       * Gets the list of blacklisted trackers along with reasons for blacklisting.
456       * 
457       * @return the collection of {@link BlackListInfo} objects. 
458       * 
459       */
460      public Collection<BlackListInfo> getBlackListedTrackersInfo() {
461        return blacklistedTrackersInfo;
462      }
463    
464      /**
465       * Get the current state of the <code>JobTracker</code>,
466       * as {@link JobTracker.State}
467       *
468       * {@link JobTracker.State} should no longer be used on M/R 2.x. The function
469       * is kept to be compatible with M/R 1.x applications.
470       *
471       * @return the invalid state of the <code>JobTracker</code>.
472       */
473      @Deprecated
474      public JobTracker.State getJobTrackerState() {
475        return JobTracker.State.RUNNING;
476      }
477    
478      public void write(DataOutput out) throws IOException {
479        if (activeTrackers.size() == 0) {
480          out.writeInt(numActiveTrackers);
481          out.writeInt(0);
482        } else {
483          out.writeInt(activeTrackers.size());
484          out.writeInt(activeTrackers.size());
485          for (String tracker : activeTrackers) {
486            Text.writeString(out, tracker);
487          }
488        }
489        if (blacklistedTrackersInfo.size() == 0) {
490          out.writeInt(numBlacklistedTrackers);
491          out.writeInt(blacklistedTrackersInfo.size());
492        } else {
493          out.writeInt(blacklistedTrackersInfo.size());
494          out.writeInt(blacklistedTrackersInfo.size());
495          for (BlackListInfo tracker : blacklistedTrackersInfo) {
496            tracker.write(out);
497          }
498        }
499        out.writeInt(numExcludedNodes);
500        out.writeLong(ttExpiryInterval);
501        out.writeInt(map_tasks);
502        out.writeInt(reduce_tasks);
503        out.writeInt(max_map_tasks);
504        out.writeInt(max_reduce_tasks);
505        WritableUtils.writeEnum(out, status);
506        out.writeInt(grayListedTrackers);
507      }
508    
509      public void readFields(DataInput in) throws IOException {
510        numActiveTrackers = in.readInt();
511        int numTrackerNames = in.readInt();
512        if (numTrackerNames > 0) {
513          for (int i = 0; i < numTrackerNames; i++) {
514            String name = StringInterner.weakIntern(Text.readString(in));
515            activeTrackers.add(name);
516          }
517        }
518        numBlacklistedTrackers = in.readInt();
519        int blackListTrackerInfoSize = in.readInt();
520        if(blackListTrackerInfoSize > 0) {
521          for (int i = 0; i < blackListTrackerInfoSize; i++) {
522            BlackListInfo info = new BlackListInfo();
523            info.readFields(in);
524            blacklistedTrackersInfo.add(info);
525          }
526        }
527        numExcludedNodes = in.readInt();
528        ttExpiryInterval = in.readLong();
529        map_tasks = in.readInt();
530        reduce_tasks = in.readInt();
531        max_map_tasks = in.readInt();
532        max_reduce_tasks = in.readInt();
533        status = WritableUtils.readEnum(in, JobTrackerStatus.class);
534        grayListedTrackers = in.readInt();
535      }
536    }